From Pipenv to Poetry (#1391)

* Add more tomls

* fix cli versioning

* Drop pipfile

* Drop pipfile

* Drop pipfile

* keep pipfile

* keep pipfile

* without pipfile

* typo

* typo

* typo

* typo

* typo

* typo

* typo

* typo

* readmes

* drop requires

* add requires

* w/o readme

* w/o readme

* w/o readme

* pdm run

* migrate to pdm

* migrate to pdm

* readme rename

* Fix versioning and docker building

* Fix versioning and docker building

* Fix versioning and docker building

* change build be

* change build be

* PDM_VENV_WITH_PIP

* source-includes

* source-includes

* source-includes

* compose logs

* compose logs

* skip hanging

* skip hanging

* skip hanging

* skip hanging

* readd test

* fix test

* remove test

* remove test

* docs and dockerfiles

* Try with poetry

* Try with poetry

* Try with poetry

* Try with poetry

* drop numpy

* version

* no-root

* fix version extraction

* docs and dockerfiles

* revert test

* comment hanging test

* try clause

.github/CODEOWNERS

@@ -1,7 +1,2 @@
-# Package dependency changes should be approved by a member of 'presidio-administrators' team
-*/Pipfile* @microsoft/presidio-administrators
-
-# Version change should be approved by a member of 'presidio-administrators' team
-VERSION @microsoft/presidio-administrators
-VERSION-IMAGE-REDACTOR @microsoft/presidio-administrators
-
+# Package dependencies and version changes should be approved by a member of 'presidio-administrators' team
+**/pyproject.toml @microsoft/presidio-administrators

.gitignore

@@ -90,7 +90,7 @@ ENV/
 env.bak/
 venv.bak/
 *venv/
-*Pipfile.lock
+poetry.lock
 
 # Spyder project settings
 .spyderproject

.pipelines/templates/build-analyzer.yml

@@ -1,13 +1,12 @@
 steps:
   - task: Bash@3
-    displayName: 'Setup pipenv'
+    displayName: 'Setup poetry'
     inputs:
       targetType: 'inline'
       script: |
         set -eux  # fail on error
         python -m pip install --upgrade pip
-        python -m pip install pipenv
-        pipenv --python 3
+        python -m pip install poetry
 
   - task: Bash@3
     displayName: 'Install deps'
@@ -16,9 +15,9 @@ steps:
       workingDirectory: 'presidio-analyzer'
       script: |
         set -eux  # fail on error
-        pipenv install --deploy --dev
-        pipenv run python -m spacy download en_core_web_lg
-        pipenv run python -m spacy download en_core_web_sm
+        poetry install --all-extras
+        poetry run python -m spacy download en_core_web_lg
+        poetry run python -m spacy download en_core_web_sm
 
   - template: ./build-python.yml
     parameters:

.pipelines/templates/build-anonymizer.yml

@@ -1,13 +1,12 @@
 steps:
   - task: Bash@3
-    displayName: 'Setup pipenv'
+    displayName: 'Setup poetry'
     inputs:
       targetType: 'inline'
       script: |
         set -eux  # fail on error
         python -m pip install --upgrade pip
-        python -m pip install pipenv
-        pipenv --python 3
+        python -m pip install poetry
 
   - task: Bash@3
     displayName: 'Install deps: Anonymizer'
@@ -16,7 +15,7 @@ steps:
       workingDirectory: 'presidio-anonymizer'
       script: |
         set -eux  # fail on error
-        pipenv install --deploy --dev
+        poetry install
 
   - template: ./build-python.yml
     parameters:

.pipelines/templates/build-cli.yml

@@ -1,22 +1,22 @@
 steps:
   - task: Bash@3
-    displayName: 'Setup pipenv'
+    displayName: 'Setup poetry'
     inputs:
       targetType: 'inline'
       script: |
         set -eux  # fail on error
         python -m pip install --upgrade pip
-        python -m pip install pipenv
-        pipenv --python 3
+        python -m pip install poetry
+
   - task: Bash@3
     displayName: 'Install deps'
     inputs:
       targetType: 'inline'
       workingDirectory: 'presidio-cli'
       script: |
         set -eux  # fail on error
-        pipenv install --deploy --dev
-        pipenv run python -m spacy download en_core_web_lg
+        poetry install
+        poetry run python -m spacy download en_core_web_lg
   - template: ./build-python.yml
     parameters:
       SERVICE: 'Cli'

.pipelines/templates/build-image-redactor.yml

@@ -10,24 +10,24 @@ steps:
               sudo apt show tesseract-ocr
               sudo tesseract -v
     - task: Bash@3
-      displayName: 'Setup pipenv'
+      displayName: 'Setup poetry'
       inputs:
           targetType: 'inline'
           script: |
               set -eux  # fail on error
               python -m pip install --upgrade pip
-              python -m pip install pipenv
-              pipenv --python 3
+              python -m pip install poetry
+
     - task: Bash@3
       displayName: 'Install deps'
       inputs:
           targetType: 'inline'
           workingDirectory: 'presidio-image-redactor'
           script: |
               set -eux  # fail on error
-              pipenv install --deploy --dev
-              pipenv run python -m spacy download en_core_web_lg
-              pipenv run pip install -e ../presidio-analyzer/.
+              poetry install
+              poetry run python -m spacy download en_core_web_lg
+              poetry run pip install -e ../presidio-analyzer/.
     - template: ./build-python.yml
       parameters:
           SERVICE: 'Image-Redactor'

.pipelines/templates/build-python.yml

@@ -12,8 +12,8 @@ steps:
           script: |
               set -eux  # fail on error
               # Install pytest and run tests
-              pipenv run pip install pytest pytest-azurepipelines
-              pipenv run pytest -vv
+              poetry run pip install pytest pytest-azurepipelines
+              poetry run pytest -vv
 
     - task: Bash@3
       displayName: 'Package Wheel: ${{ parameters.SERVICE }}'

.pipelines/templates/build-structured.yml

@@ -1,13 +1,12 @@
 steps:
   - task: Bash@3
-    displayName: 'Setup pipenv'
+    displayName: 'Setup poetry'
     inputs:
       targetType: 'inline'
       script: |
         set -eux  # fail on error
         python -m pip install --upgrade pip
-        python -m pip install pipenv
-        pipenv --python 3
+        python -m pip install poetry
 
   - task: Bash@3
     displayName: 'Install deps'
@@ -16,9 +15,9 @@ steps:
       workingDirectory: 'presidio-structured'
       script: |
         set -eux  # fail on error
-        pipenv install --deploy --dev
-        pipenv run pip install -e ../presidio-analyzer/. # Use the existing analyzer and not the one in PyPI
-        pipenv run pip install -e ../presidio-anonymizer/. # Use the existing analyzer and not the one in PyPI
+        poetry install
+        poetry run pip install -e ../presidio-analyzer/. # Use the existing analyzer and not the one in PyPI
+        poetry run pip install -e ../presidio-anonymizer/. # Use the existing analyzer and not the one in PyPI
 
   - template: ./build-python.yml
     parameters:

.pipelines/templates/e2e-tests.yml

@@ -46,3 +46,10 @@ steps:
           ANONYMIZER_BASE_URL: ${{ parameters.anonymizer_base_url }}
         TEST_SUITE:  ${{ parameters.test_suite }}
       displayName: Run tests
+    - task: DockerCompose@0
+      displayName: Docker Logs
+      inputs:
+        dockerComposeCommand: logs
+        dockerComposeFile: docker-compose.yml
+        buildImages: false
+      condition: always()

.pipelines/templates/release.yml

@@ -14,7 +14,7 @@ stages:
       steps:
         - bash: |
             set -eu  # exit on error
-            ver=$(cat VERSION)
+            ver=$(grep -m 1 version presidio-analyzer/pyproject.toml | tr -s ' ' | tr -d '"' | tr -d "'" | cut -d' ' -f3)
             echo $ver
             echo "##vso[task.setvariable variable=version;isOutput=true]$ver"
           displayName: Set Version
@@ -25,7 +25,7 @@ stages:
       steps:
         - bash: |
             set -eu  # exit on error
-            imageVer=$(cat VERSION-IMAGE-REDACTOR)
+            imageVer=$(grep -m 1 version presidio-image-redactor/pyproject.toml | tr -s ' ' | tr -d '"' | tr -d "'" | cut -d' ' -f3)
             echo $imageVer
             echo "##vso[task.setvariable variable=imageVersion;isOutput=true]$imageVer"
           displayName: Set Image Version

docs/development.md

@@ -16,66 +16,52 @@ The project is structured so that:
 - In the project root directory, you will find common code for using, serving and testing Presidio
     as a cluster of services, as well as CI/CD pipelines codebase and documentation.
 
-### Setting up Pipenv
+### Setting up Poetry
 
-[Pipenv](https://pipenv.pypa.io/en/latest/) is a Python workflow manager, handling
-dependencies and environment for Python packages. It is used by each Presidio service
-as the dependencies manager, to be aligned with the specific requirements versions.
-Follow these steps when starting to work on a Presidio service with Pipenv:
+[Poetry](https://python-poetry.org/) is Python package manager. It is used to manage dependencies and virtual
+environments for Presidio services.
+Follow these steps when starting to work on a Presidio service with poetry:
 
-1. Install Pipenv
+1. Install poetry
 
     - Using Pip
 
         ```sh
-        pip install --user pipenv
+        pip install poetry
         ```
 
     - Using Homebrew (in MacOS)
 
         ```
-        brew install pipenv
+        brew install poetry
         ```
 
-    Additional installation instructions for Pipenv: <https://pipenv.readthedocs.io/en/latest/install/#installing-pipenv>
+    Additional installation instructions for poetry: <https://python-poetry.org/docs/#installation>
 
-2. Have Pipenv create a virtualenv for the project and install all requirements in the Pipfile,
+2. Have poetry create a virtualenv for the project and install all requirements in the pyproject.toml,
     including dev requirements.
 
     For example, in the `presidio-analyzer` folder, run:
 
     ```
-    pipenv install --dev --skip-lock
+    poetry install --all-extras
     ```
 
 3. Run all tests:
 
     ```
-    pipenv run pytest
+    poetry run pytest
     ```
 
 4. To run arbitrary scripts within the virtual env, start the command with
-    `pipenv run`. For example:
-    1. `pipenv run ruff check`
-    2. `pipenv run pip freeze`
-    3. `pipenv run python -m spacy download en_core_web_lg`
+    `poetry run`. For example:
+    1. `poetry run ruff check`
+    2. `poetry run pip freeze`
+    3. `poetry run python -m spacy download en_core_web_lg`
 
     Command 3 downloads the default spacy model needed for Presidio Analyzer.`
 
-#### Alternatively, activate the virtual environment and use the commands by starting a pipenv shell
-
-1. Start shell:
-
-    ```
-    pipenv shell
-    ```
-
-2. Run commands in the shell
-
-    ```
-    pytest
-    pip freeze
-    ```
+#### Alternatively, activate the virtual environment and use the commands using [this method](https://python-poetry.org/docs/basic-usage/#activating-the-virtual-environment).
 
 ### Development guidelines
 
@@ -107,9 +93,9 @@ use docker-compose ps:
 ```bash
 >docker-compose ps
 CONTAINER ID   IMAGE                       COMMAND                  CREATED         STATUS         PORTS                    NAMES
-6d5a258d19c2   presidio-anonymizer         "/bin/sh -c 'pipenv …"   6 minutes ago   Up 6 minutes   0.0.0.0:5001->5001/tcp   presidio_presidio-anonymizer_1
-9aad2b68f93c   presidio-analyzer           "/bin/sh -c 'pipenv …"   2 days ago      Up 6 minutes   0.0.0.0:5002->5001/tcp   presidio_presidio-analyzer_1
-1448dfb3ec2b   presidio-image-redactor     "/bin/sh -c 'pipenv …"   2 seconds ago   Up 2 seconds   0.0.0.0:5003->5001/tcp   presidio_presidio-image-redactor_1
+6d5a258d19c2   presidio-anonymizer         "/bin/sh -c 'poetry …"   6 minutes ago   Up 6 minutes   0.0.0.0:5001->5001/tcp   presidio_presidio-anonymizer_1
+9aad2b68f93c   presidio-analyzer           "/bin/sh -c 'poetry …"   2 days ago      Up 6 minutes   0.0.0.0:5002->5001/tcp   presidio_presidio-analyzer_1
+1448dfb3ec2b   presidio-image-redactor     "/bin/sh -c 'poetry …"   2 seconds ago   Up 2 seconds   0.0.0.0:5003->5001/tcp   presidio_presidio-image-redactor_1
 ```
 
 Edit docker-compose.yml configuration file to change the default ports.
@@ -152,7 +138,7 @@ Running the tests locally can be done in two ways:
 1. Using cli, from each service directory, run:
 
     ```sh
-    pipenv run pytest
+    poetry run pytest
     ```
 
 2. Using your IDE.
@@ -235,7 +221,7 @@ run.bat
 
 Presidio services are PEP8 compliant and continuously enforced on style guide issues during the build process using `ruff`, in turn running `flake8` and other linters.
 
-Running ruff locally, using `pipenv run ruff check`, you can check for those issues prior to committing a change.
+Running ruff locally, using `poetry run ruff check`, you can check for those issues prior to committing a change.
 
 Ruff runs linters in addition to the basic `flake8` functionality, Presidio uses linters as part as ruff such as: