// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

syntax = "proto3";

package google.cloud.visualinspection.v1beta1;

import "google/api/annotations.proto";
import "google/api/field_behavior.proto";

option csharp_namespace = "Google.Cloud.VisualInspection.V1Beta1";
option php_namespace = "Google\\Cloud\\VisualInspection\\V1beta1";
option ruby_package = "Google::Cloud::VisualInspection::V1beta1";
option go_package = "google.golang.org/genproto/googleapis/cloud/visualinspection/v1beta1;visualinspection";
option java_multiple_files = true;
option java_package = "com.google.cloud.visualinspection.v1beta1";

// All metrics for the solution.
message Metrics {
  // Output only. Metrics for the solution at different slices.
  map<string, MetricsSlice> metrics_slice = 1
      [(google.api.field_behavior) = OUTPUT_ONLY];
}

// Metrics at a particular slice.
// A slice can be "pixel" level, "image" level, "component" level, "unit" level,
// and/or "product" level depending on the solution and the input data.
message MetricsSlice {
  // Model metrics at a specific threshold.
  message ConfidenceLevelModelMetrics {
    // Output only. Model metrics.
    ModelMetrics model_metrics = 1 [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. Confidence threshold in the range [0, 1].
    float confidence_threshold = 2 [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. Pixel level confidence threshold in the range [0, 1].
    float pixel_confidence_threshold = 3
        [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. Cluster distance threshold in the range [0, 1] normalized by
    // the diagonal length of the image.
    float cluster_distance_threshold = 4
        [(google.api.field_behavior) = OUTPUT_ONLY];
  }

  // Output only. The AnnotationSpec which this MetricSlice refers to. It is
  // empty if this MetricSlice aggregates across AnnotationSpecs.
  string annotation_spec = 8 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The AnnotationSpec display name which this MetricSlice refers
  // to. It is empty if this MetricSlice aggregates across AnnotationSpecs.
  string display_name = 9 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The number of examples used to create this evaluation metrics
  // instance.
  int32 evaluated_example_count = 10
      [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Model metrics across confidence thresholds.
  repeated ConfidenceLevelModelMetrics confidence_level_model_metrics = 5
      [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Average model metrics.
  ModelMetrics average_model_metrics = 7
      [(google.api.field_behavior) = OUTPUT_ONLY];
}

// Model metrics which can vary depending on the model. These are not assumed to
// refer to defect states in particular.
message ModelMetrics {
  // Output only. The false negative rate in the range [0, 1]. This is the rate
  // with which the model incorrectly classifies examples as negative when they
  // are positive. This is consistent with the following definition:
  //
  // https:
  // //en.wikipedia.org/wiki/False_positives_
  // // and_false_negatives#False_positive_and_false_negative_rates.
  float false_negative_rate = 1 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The false positive rate in the range [0, 1]. This is the rate
  // with which the model  incorrectly classifies examples as positive when they
  // are negative. This is consistent with the following definition:
  //
  // https:
  // //en.wikipedia.org/wiki/False_positives_
  // // and_false_negatives#False_positive_and_false_negative_rates.
  float false_positive_rate = 2 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The precision in the range [0, 1]. Precision measures the
  // ratio of correctly classified examples by the model against all the
  // predictions made by the model. Please see
  // https://en.wikipedia.org/wiki/Precision_and_recall for its definition.
  float precision = 3 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The recall in the range [0, 1]. Recall measures the ratio of
  // correctly predicted labels by the model against all true examples of that
  // label. Please see https://en.wikipedia.org/wiki/Precision_and_recall for
  // its definition.
  float recall = 4 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The Intersection over union (IOU) in the range [0, 1]. IOU
  // measures the proportion of label regions that are common between ground
  // truth and model predictions versus their union. Please see
  // https://en.wikipedia.org/wiki/Jaccard_index for its definition.
  float iou = 5 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The Dice similarity coefficient in the range [0, 1]. Dice
  // similarity coefficient is similar to IOU except that the common label
  // region is weighted twice in the calculation. Please see its definition
  // here: https://en.wikipedia.org/wiki/S%C3%B8rensen%E2%80%93Dice_coefficient
  float dice_similarity_coefficient = 6
      [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Confusion matrix for the classifications.
  // Only present for aggregated metrics.
  ConfusionMatrix confusion_matrix = 7
      [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The average precision in the range [0, 1].
  float average_precision = 8 [(google.api.field_behavior) = OUTPUT_ONLY];
}

// Confusion matrix across different class labels (if present).
message ConfusionMatrix {
  // Individual row for the confusion matrix.
  message Row {
    // Output only. Count of items in a "slice" classified at a particular
    // class_label in the ground truth versus that classified by the solution.
    repeated int64 count = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
  }

  // Output only. Counts of items across slices.
  // For example, rows[i][j].count indicate the count for class_labels[i] as
  // present in ground truth versus class_labels[j] as predicted by the model.
  // The number of rows is the same as individual number of "counts" in a Row as
  // well as the number of class_labels. So if the number of class_labels is n,
  // then rows count is n as well as each rows.count has also count n.
  repeated Row rows = 1 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Class labels as the rows/columns of the confusion matrix.
  repeated string class_labels = 2 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Class AnnotationSpecs as the rows/columns of the confusion
  // matrix.
  repeated string annotation_specs = 3
      [(google.api.field_behavior) = OUTPUT_ONLY];
}
