dev -> main pour version 1.0.2

.github/workflows/cicd_deploy.yml

@@ -0,0 +1,74 @@
+
+name: cicd_deploy
+
+on:
+  # Also run when the pull request merges (which generates a push)
+  # So that we can tag the docker image appropriately.
+  push:
+    tags: [ 'v*.*.*' ]
+
+env:
+  IMAGE_NAME:  ${{ github.repository }}
+  REGISTRY: ghcr.io
+  TEST_TAG:  ${{ github.repository }}:test
+
+jobs:
+  deploy_docker:
+    runs-on: ubuntu-latest
+
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+    - name: Checkout branch
+      uses: actions/checkout@v4
+
+    - name: Build the Docker image
+      id: build
+      uses: docker/build-push-action@v6
+      with:
+        load: true
+        tags: ${{ env.TEST_TAG }}
+
+    # run the test on the docker image
+    - name: Run tests in docker image
+      run: >
+        docker run
+        ${{ env.TEST_TAG }}
+        python -m pytest -s
+        --log-cli-level=DEBUG
+        --log-format="%(asctime)s %(levelname)s %(message)s"
+        --log-date-format="%Y-%m-%d %H:%M:%S"
+        -m "not functional_test"
+
+    # Login against a Docker registry except on PR
+    # https://github.com/docker/login-action
+    - name: Log into registry ${{ env.REGISTRY }}
+      if: github.event_name != 'pull_request'
+      uses: docker/login-action@v3.4
+      with:
+        registry: ${{ env.REGISTRY }}
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+
+    # Extract metadata (tags, labels) for Docker
+    # https://github.com/docker/metadata-action
+    - name: Extract Docker metadata
+      if: github.event_name != 'pull_request'
+      id: meta
+      uses: docker/metadata-action@v5
+      with:
+        images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+    # Build a Docker image with Buildx (don't on PR)
+    # https://github.com/docker/build-push-action
+    - name: Build and push Docker image
+      if: github.event_name != 'pull_request'
+      id: build-and-push
+      uses: docker/build-push-action@v6
+      with:
+        context: .
+        push: true
+        tags: ${{ steps.meta.outputs.tags }}
+        labels: ${{ steps.meta.outputs.labels }}

.github/workflows/cicd_full.yml

@@ -0,0 +1,69 @@
+
+name: cicd_full
+
+on:
+  # Run tests for pull-requests on master
+  pull_request:
+    branches:
+      - main
+  # Also run when the pull request merges  (which generates a push)
+  # So that we can tag the docker image appropriately.
+  push:
+    branches:
+      - dev
+
+env:
+  IMAGE_NAME:  ${{ github.repository }}
+  REGISTRY: ghcr.io
+  TEST_TAG:  ${{ github.repository }}:test
+
+jobs:
+  deploy_docker:
+    runs-on: ubuntu-latest
+
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+    - name: Checkout branch
+      uses: actions/checkout@v4
+
+    - name: Build the Docker image
+      id: build
+      uses: docker/build-push-action@v6
+      with:
+        load: true
+        tags: ${{ env.TEST_TAG }}
+
+    # run the test on the docker image
+    - name: Run tests in docker image
+      run: >
+        docker run
+        ${{ env.TEST_TAG }}
+        python -m pytest -s
+        --log-cli-level=DEBUG
+
+  test_local:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout branch
+        uses: actions/checkout@v4
+
+      # See https://github.com/marketplace/actions/setup-micromamba
+      - name: setup-micromamba
+        uses: mamba-org/setup-micromamba@v2
+        with:
+          environment-file: environment.yml
+          environment-name: ctview  # activate the environment
+          cache-environment: true
+          cache-downloads: true
+          generate-run-shell: true
+
+      - name: Run tests with pytest
+        shell: micromamba-shell {0}
+        run: python -m pytest -s --log-cli-level=DEBUG

.github/workflows/cicd_light.yml

@@ -0,0 +1,36 @@
+name: cicd_light
+
+on:
+  # Run tests for non-draft pull request on dev
+  pull_request:
+    branches:
+      - dev
+
+
+jobs:
+  test_light:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout branch
+        uses: actions/checkout@v4
+
+      # See https://github.com/marketplace/actions/setup-micromamba
+      - name: setup-micromamba
+        uses: mamba-org/setup-micromamba@v2
+        with:
+          environment-file: environment.yml
+          environment-name: ctview  # activate the environment
+          cache-environment: true
+          cache-downloads: true
+          generate-run-shell: true
+
+      - name: Run tests with pytest
+        shell: micromamba-shell {0}
+        run: python -m pytest -s --log-cli-level=DEBUG
+
+
+

Dockerfile

@@ -11,12 +11,11 @@
 
 # # Set up the Conda environment: cf https://github.com/mamba-org/micromamba-docker
 COPY environment.yml /tmp/env.yaml
-COPY requirements.txt /tmp/requirements.txt
 RUN chown $MAMBA_USER:$MAMBA_USER /tmp/env.yaml
 RUN micromamba install -y -n base -f /tmp/env.yaml && \
     micromamba clean --all --yes
 
 ENV ENV_NAME base
 ARG MAMBA_DOCKERFILE_ACTIVATE=1
 
 COPY ctview ctview

Makefile

@@ -1,3 +1,9 @@
+# Makefile to manage main tasks
+# cf. https://blog.ianpreston.ca/conda/python/bash/2020/05/13/conda_envs.html#makefile
+
+# Oneshell means I can run multiple lines in a recipe in the same shell, so I don't have to
+# chain commands together with semicolon
+.ONESHELL:
 
 install:
 	mamba env update -n ctview -f environment.yml
@@ -6,27 +12,27 @@ install-precommit:
 	pre-commit install
 
 testing:
-	./ci/test.sh
+	python -m pytest -s ./test -v
 
 ##############################
 # Docker
 ##############################
 
-PROJECT_NAME=lidar_hd/ct_view
+REGISTRY=ghcr.io
+NAMESPACE=ignf
+IMAGE_NAME=ctview
 VERSION=`python -m ctview._version`
-REGISTRY=docker-registry.ign.fr
+FULL_IMAGE_NAME=${REGISTRY}/${NAMESPACE}/${IMAGE_NAME}:${VERSION}
 
 docker-build:
-	docker build -t ${PROJECT_NAME}:${VERSION} -f Dockerfile .
+	docker build -t ${IMAGE_NAME}:${VERSION} -f Dockerfile .
 
 docker-test:
-	docker run --rm -it ${PROJECT_NAME}:${VERSION} python -m pytest -s
+	docker run --rm ${IMAGE_NAME}:${VERSION} python -m pytest -s -m "not functional_test"
 
 docker-remove:
-	docker rmi -f `docker images | grep ${PROJECT_NAME} | tr -s ' ' | cut -d ' ' -f 3`
-	docker rmi -f `docker images -f "dangling=true" -q`
+	docker rmi -f `docker images | grep ${IMAGE_NAME}:${VERSION} | tr -s ' ' | cut -d ' ' -f 3`
 
 docker-deploy:
-	docker login docker-registry.ign.fr -u svc_lidarhd
-	docker tag ${PROJECT_NAME}:${VERSION} ${REGISTRY}/${PROJECT_NAME}:${VERSION}
-	docker push ${REGISTRY}/${PROJECT_NAME}:${VERSION}
+	docker tag ${IMAGE_NAME}:${VERSION} ${FULL_IMAGE_NAME}
+	docker push ${FULL_IMAGE_NAME}

README.md

@@ -1,6 +1,33 @@
 # CtView
 
-Ce logiciel permet de créer différents rasters représentant les données contenues dans un nuage de points. Il a été initiallement développé pour les usages suivants :
+# English version (short)
+
+## Overview
+CtView is a tool based on pdal/gdal that generates misc thematic maps (raster views) from las/laz files.
+
+Its initial goal was to create
+- operator views for pointcloud classification control
+- raster metadata (density and classification maps)
+
+The currently implemented rasters are:
+- density map
+- classification map: in case of a pixel that contains several classes, classification values can be combined in combination classes (provided as parameters), and a priority list is used to choose the class to display among the remaining classes
+
+Rasters can be (or not) colorized or shadowed using a digital surface/terrain model.
+
+## Usage
+
+- Ctview has been developed using Linux Ubuntu and tested under linux and windows
+- CtView should be used as a command line tool (commands must be executed from ctview's root folder)
+- It is intended to run on a single file (to use it on a folder, an external loop is required)
+- The configuration is handled via yaml files with [hydra](https://hydra.cc/docs/intro/)
+- A few commands for installation and tests are provided in a [Makefile](Makefile) in the root folder
+- conda is used for environment building, pytest for testing
+- The entry point of the code is [ctview/main_ctview.py](ctview/main_ctview.py)
+
+# Version française (plus complète)
+
+Ce logiciel permet de créer différents rasters représentant les données contenues dans un nuage de points. Il a été initialement développé pour les usages suivants :
 - Créer des vues opérateurs pour du contrôle de classification de nuages de points.
 - Créer des métadonnées associées au nuage de points (carte de densité et cartes de classes)
 
@@ -134,9 +161,3 @@ exemple_control_fast.sh
 ```
 exemple_metadata_fast.sh
 ```
-
-# POC Aymeric metadata carte de classe
-
-```
-/var/data/store-echange/SV3D/Aymeric/RDI/DEMO_Raster_Class
-```

RELEASE_NOTES.md

@@ -1,3 +1,12 @@
+# dev
+
+# v1.0.2
+- Fix bug 2461, due to an error on ZIP compression while making buffered tile.
+  The temporary buffered tile is now a LAS (instead of a LAZ), so there is no compression.
+- Adapt ci for github
+- host docker image on ghcr.io
+- update dependencies
+
 # v1.0.1
 - Fix tile origin detection on tiles that are thinner than the buffer size
 - Make density and class map computation optional by setting the corresponding output sub-dirs to "null"

configs/config_control.yaml

@@ -88,7 +88,7 @@ class_map:
   - {value: 202, description: "Candidats Batiment", color: [255, 128, 128]}
   dxm_filter:  # Paramètres pour le calcul du MNS (utilisé pour l'ombrage), dimension à filtrer et
                # valeur à garder, par défaut on utilise la dimension créée dans la phase de preprocessing
-               # des produits dérivés
+               # de las_digital_models
     dimension: dsm_marker
     keep_values: [1]
   post_processing:  # Traitements à appliquer à la carte de classe

configs/config_metadata.yaml

@@ -96,7 +96,7 @@ class_map:
     - {value: 162, description: "VirtPont", color: [85, 97, 137]}
   dxm_filter:  # Paramètres pour le calcul du MNS (utilisé pour l'ombrage), dimension à filtrer et
                # valeur à garder, par défaut on utilise la dimension créée dans la phase de preprocessing
-               # des produits dérivés
+               # de las_digital_models
     dimension: dsm_marker
     keep_values: [1]
   post_processing:  # Traitements à appliquer à la carte de classe

ctview/_version.py

@@ -1,4 +1,4 @@
-__version__ = "1.0.1"
+__version__ = "1.0.2"
 
 if __name__ == "__main__":
     print(__version__)

ctview/main_ctview.py

@@ -48,12 +48,15 @@ def main_ctview(config: DictConfig):
 
         # Buffer
         log.info(f"\nStep 1: Create buffered las file with buffer = {config.buffer.size}")
+        # Use .las extension (instead of .laz) to avoid a bug with LAZ compression
+        buffered_las = Path(initial_las_filename).stem + ".las"
         if config.buffer.output_subdir:
-            las_with_buffer = Path(out_dir) / config.buffer.output_subdir / initial_las_filename
+            las_with_buffer = Path(out_dir) / config.buffer.output_subdir / buffered_las
         else:
-            las_with_buffer = Path(tmpdir_buffer) / initial_las_filename
-        las_with_buffer.parent.mkdir(parents=True, exist_ok=True)
+            las_with_buffer = Path(tmpdir_buffer) / buffered_las
 
+        las_with_buffer.parent.mkdir(parents=True, exist_ok=True)
+        print("las with buffer:", las_with_buffer)
         epsg = config.io.spatial_reference
         create_las_with_buffer(
             input_dir=str(in_dir),