Release v10.11.4

.github/workflows/deploy-custom-prod.yml

@@ -23,7 +23,8 @@ jobs:
             default_application_status: APPROVED
             cost_profile: DEFAULT
             job_files: >-
-              job_spec/AUTORIFT_ITS_LIVE.yml
+              job_spec/ITS_LIVE_AUTORIFT.yml
+              job_spec/ITS_LIVE_META.yml
             instance_types: r7gd.2xlarge,r7gd.4xlarge,r7gd.8xlarge
             default_max_vcpus: 2000  # Max: 10,406
             expanded_max_vcpus: 2000  # Max: 10,406
@@ -140,8 +141,8 @@ jobs:
               job_spec/INSAR_ISCE.yml
               job_spec/OPERA_DIST_S1.yml
             instance_types: m6id.xlarge,m6id.2xlarge,m6id.4xlarge,m6id.8xlarge,m6idn.xlarge,m6idn.2xlarge,m6idn.4xlarge,m6idn.8xlarge
-            default_max_vcpus: 1600  # Max 1652
-            expanded_max_vcpus: 1600  # Max 1652
+            default_max_vcpus: 1600  # Max: 1652
+            expanded_max_vcpus: 1600  # Max: 1652
             required_surplus: 0
             security_environment: JPL-public
             ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id
@@ -237,8 +238,8 @@ jobs:
               job_spec/INSAR_ISCE_BURST.yml
               job_spec/INSAR_ISCE_MULTI_BURST.yml
             instance_types: r6id.xlarge,r6id.2xlarge,r6id.4xlarge,r6id.8xlarge,r6idn.xlarge,r6idn.2xlarge,r6idn.4xlarge,r6idn.8xlarge
-            default_max_vcpus: 1600
-            expanded_max_vcpus: 1600
+            default_max_vcpus: 6000  # Max: 6000
+            expanded_max_vcpus: 6000  # Max: 6000
             required_surplus: 0
             security_environment: ASF
             ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id

.github/workflows/deploy-custom-test.yml

@@ -64,7 +64,8 @@ jobs:
             default_application_status: APPROVED
             cost_profile: DEFAULT
             job_files: >-
-              job_spec/AUTORIFT_ITS_LIVE.yml
+              job_spec/ITS_LIVE_AUTORIFT.yml
+              job_spec/ITS_LIVE_META.yml
             instance_types: r7gd.2xlarge,r7gd.4xlarge,r7gd.8xlarge
             default_max_vcpus: 640
             expanded_max_vcpus: 640

CHANGELOG.md

@@ -4,6 +4,19 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [10.11.4]
+
+### Added
+- Added METADATA step to `ITS_LIVE_AUTORIFT` to generate a project STAC JSON file and other metadata files for ingest into NSIDC DAAC.
+- Added a new `ITS_LIVE_META` job spec which generates STAC JSON and other metadata files for already published ITS_LIVE products.
+- Added the `ITS_LIVE_META` job spec to the ITS_LIVE deployments.
+
+### Changed
+- Renamed the `AUTORIFT_ITS_LIVE` job spec to `ITS_LIVE_AUTORIFT` to better group the ITS_LIVE project specific job specs.
+- Increase throughput for Cargill deployment by increasing max vCPUs to 6000 from 1600.
+- OPERA DIST-S1 default parameters and memory allocated.
+- Updated OPERA DIST-S1 command to match the entrypoint in the associated plugin (docker container).
+
 ## [10.11.3]
 
 ### Added

job_spec/AUTORIFT_ITS_LIVE.yml → job_spec/ITS_LIVE_AUTORIFT.yml

@@ -116,6 +116,11 @@ AUTORIFT:
           - "its-live-data"
           - "its-live-data-test"
         default: null
+    publish_stac_prefix:
+      api_schema:
+        description: Publish the resulting STAC JSON item under this prefix in the ITS_LIVE AWS Open Data (or test) S3 Bucket
+        type: string
+        default: test-space/stac/ndjson/ingest
     use_static_files:
       api_schema:
         description:  Use static topographic correction files for ISCE3 processing if available (Sentinel-1 only)
@@ -155,3 +160,21 @@ AUTORIFT:
         - EARTHDATA_PASSWORD
         - PUBLISH_ACCESS_KEY_ID
         - PUBLISH_SECRET_ACCESS_KEY
+    - name: METADATA
+      image: ghcr.io/asfhyp3/itslive-metadata
+      command:
+        - --bucket
+        - '!Ref Bucket'
+        - --bucket-prefix
+        - Ref::job_id
+        - --publish-bucket
+        - Ref::publish_bucket
+        - --publish-prefix
+        - Ref::publish_stac_prefix
+      timeout: 10800
+      compute_environment: ItsLiveMeta
+      vcpu: 1
+      memory: 7875
+      secrets:
+        - PUBLISH_ACCESS_KEY_ID
+        - PUBLISH_SECRET_ACCESS_KEY

job_spec/ITS_LIVE_META.yml

@@ -0,0 +1,49 @@
+ITS_LIVE_META:
+  required_parameters:
+    - granule_uri
+  parameters:
+    granule_uri:
+      api_schema:
+        description: S3 URI for an ITS_LIVE velocity granule to generate metadata for
+        type: string
+        pattern: '^s3:\/\/.*\.nc$'
+    publish_bucket:
+      api_schema:
+        description: Publish the resulting metadata files to the ITS_LIVE AWS Open Data (or test) S3 Bucket
+        type: string
+        nullable: true
+        enum:
+          - null
+          - "its-live-data"
+          - "its-live-data-test"
+        default: null
+    publish_stac_prefix:
+      api_schema:
+        description: Publish the resulting STAC JSON item under this prefix in the ITS_LIVE AWS Open Data (or test) S3 Bucket
+        type: string
+        default: test-space/stac/ndjson/ingest
+  cost_profiles:
+    DEFAULT:
+      cost: 1.0
+  validators: []
+  steps:
+    - name: ''
+      image: ghcr.io/asfhyp3/itslive-metadata
+      command:
+        - --granule-uri
+        - Ref::granule_uri
+        - --bucket
+        - '!Ref Bucket'
+        - --bucket-prefix
+        - Ref::job_id
+        - --publish-bucket
+        - Ref::publish_bucket
+        - --publish-prefix
+        - Ref::publish_stac_prefix
+      timeout: 10800
+      compute_environment: ItsLiveMeta
+      vcpu: 1
+      memory: 7875
+      secrets:
+        - PUBLISH_ACCESS_KEY_ID
+        - PUBLISH_SECRET_ACCESS_KEY

job_spec/OPERA_DIST_S1.yml

@@ -38,27 +38,29 @@ OPERA_DIST_S1:
       api_schema:
         description: Memory strategy to use for GPU inference. Options are high, low
         type: string
-        enum: ["high", "low"]
-        default: "high"
-        example: "high"
+        enum:
+          - high
+          - low
+        default: high
+        example: high
     low_confidence_alert_threshold:
       api_schema:
         description: Low confidence alert threshold
         type: number
         format: float
-        default: 3.5
+        default: 2.5
         minimum: 0.0
         maximum: 15.0
-        example: 3.5
+        example: 2.5
     high_confidence_alert_threshold:
       api_schema:
         description: High confidence alert threshold
         type: number
         format: float
-        default: 5.5
+        default: 4.5
         minimum: 0.0
         maximum: 15.0
-        example: 5.5
+        example: 4.5
     delta_lookback_days_mw:
       api_schema:
         description: Delta lookback days for each window relative to post-image acquisition date. Comma-separated list of integers (e.g., 730,365,0) or "none" to use defaults
@@ -82,9 +84,13 @@ OPERA_DIST_S1:
       api_schema:
         description: Device to use for transformer model inference of normal parameters
         type: string
-        enum: ["cpu", "cuda", "mps", "best"]
-        default: "best"
-        example: "best"
+        enum:
+          - cpu
+          - cuda
+          - mps
+          - best
+        default: best
+        example: best
     n_workers_for_norm_param_estimation:
       api_schema:
         description: Number of CPUs to use for normal parameter estimation
@@ -96,17 +102,23 @@ OPERA_DIST_S1:
       api_schema:
         description: What model to load; external means load model from cfg and wts paths specified in parameters
         type: string
-        enum: ["external", "transformer_optimized"]
-        default: "transformer_optimized"
-        example: "transformer_optimized"
+        enum:
+          - transformer_original
+          - transformer_optimized
+          - transformer_optimized_fine
+          - transformer_anniversary_trained
+          - transformer_anniversary_trained_optimized
+          - transformer_anniversary_trained_optimized_fine
+        default: transformer_optimized
+        example: transformer_optimized
     stride_for_norm_param_estimation:
       api_schema:
         description: Number of pixels the convolutional filter moves across the input image at each step
         type: integer
-        default: 16
+        default: 7
         minimum: 1
         maximum: 16
-        example: 16
+        example: 7
     batch_size_for_norm_param_estimation:
       api_schema:
         description: Batch size for norm param estimation; Tune it according to resources i.e. memory
@@ -120,7 +132,6 @@ OPERA_DIST_S1:
         type: boolean
         default: false
         example: false
-
     use_date_encoding:
       api_schema:
         description: Whether to use acquisition date encoding in processing
@@ -135,7 +146,6 @@ OPERA_DIST_S1:
     - name: ''
       image: ghcr.io/opera-adt/dist-s1
       command:
-        - dist-s1 run
         - --mgrs_tile_id
         - Ref::mgrs_tile_id
         - --track_number
@@ -177,7 +187,7 @@ OPERA_DIST_S1:
       timeout: 10800  # 3 hr
       compute_environment: Default
       vcpu: 1
-      memory: 7500
+      memory: 31500
       secrets:
         - EARTHDATA_USERNAME
         - EARTHDATA_PASSWORD

job_spec/config/compute_environments.yml

@@ -20,3 +20,6 @@ compute_environments:
     instance_types: m6id.xlarge,m6id.2xlarge,m6id.4xlarge,m6id.8xlarge
     allocation_type: EC2
     allocation_strategy: BEST_FIT_PROGRESSIVE
+  ItsLiveMeta:
+    instance_types: r6id.xlarge,r6id.2xlarge,r6id.4xlarge,r6id.8xlarge,r6idn.xlarge,r6idn.2xlarge,r6idn.4xlarge,r6idn.8xlarge
+    ami_id: ami-0aece254fc7c27a77  # /aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id

requirements-all.txt

@@ -5,15 +5,15 @@
 -r requirements-apps-disable-private-dns.txt
 -r requirements-apps-search-archive.txt
 -r requirements-apps-update-db.txt
-boto3==1.40.12
+boto3==1.40.21
 jinja2==3.1.6
-moto[dynamodb]==5.1.10
+moto[dynamodb]==5.1.11
 pytest==8.4.1
 PyYAML==6.0.2
 responses==0.25.8
-ruff==0.12.9
+ruff==0.12.11
 mypy==1.17.1
 setuptools==80.9.0
 setuptools_scm==9.2.0
 openapi-spec-validator==0.7.2
-cfn-lint==1.39.0
+cfn-lint==1.39.1

requirements-apps-api-binary.txt

@@ -1 +1 @@
-cryptography==45.0.6
+cryptography==45.0.7

requirements-apps-api.txt

@@ -1,4 +1,4 @@
-flask==3.1.1
+flask==3.1.2
 Flask-Cors==6.0.1
 jsonschema==4.25.1
 openapi-core==0.19.5

requirements-apps-disable-private-dns.txt

@@ -1 +1 @@
-boto3==1.40.12
+boto3==1.40.21

requirements-apps-get-files.txt

@@ -1,2 +1,2 @@
-boto3==1.40.12
+boto3==1.40.21
 ./lib/dynamo/

requirements-apps-search-archive.txt

@@ -1,4 +1,4 @@
 ./lib/dynamo/
 ./lib/lambda_logging/
 asf_enumeration==0.3.0
-asf_search==10.0.0
+asf_search==10.0.2

requirements-apps-start-execution.txt

@@ -1,3 +1,3 @@
-boto3==1.40.12
+boto3==1.40.21
 ./lib/dynamo/
 ./lib/lambda_logging/