diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..0b845d3b --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,11 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "monthly" + + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "monthly" diff --git a/.github/workflows/build_executable.yml b/.github/workflows/build_executable.yml index ed522efd..2807bcf8 100644 --- a/.github/workflows/build_executable.yml +++ b/.github/workflows/build_executable.yml @@ -1,16 +1,33 @@ -name: Build executable version of CLI +name: Build executable version of CLI and optionally upload artifact on: + workflow_dispatch: + inputs: + publish: + description: 'Upload artifacts to release' + required: false + default: false + type: boolean push: branches: - main +permissions: + contents: write + jobs: build: + name: Build on ${{ matrix.os }} (${{ matrix.mode }} mode) strategy: fail-fast: false matrix: - os: [ ubuntu-20.04, macos-11, windows-2019 ] + os: [ ubuntu-22.04, macos-15-intel, macos-15, windows-2022 ] + mode: [ 'onefile', 'onedir' ] + exclude: + - os: ubuntu-22.04 + mode: onedir + - os: windows-2022 + mode: onedir runs-on: ${{ matrix.os }} @@ -20,7 +37,7 @@ jobs: steps: - name: Run Cimon - if: matrix.os == 'ubuntu-20.04' + if: matrix.os == 'ubuntu-22.04' uses: cycodelabs/cimon-action@v0 with: client-id: ${{ secrets.CIMON_CLIENT_ID }} @@ -30,31 +47,46 @@ jobs: files.pythonhosted.org install.python-poetry.org pypi.org + uploads.github.com - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Set up Python 3.7 - uses: actions/setup-python@v4 + - name: Checkout latest release tag + if: ${{ github.event_name == 'workflow_dispatch' }} + run: | + LATEST_TAG=$(git describe --tags `git rev-list --tags --max-count=1`) + git checkout $LATEST_TAG + echo "LATEST_TAG=$LATEST_TAG" >> $GITHUB_ENV + + - name: Set up Python 3.13 + uses: actions/setup-python@v6 with: - python-version: '3.7' + python-version: '3.13' + + - name: Load cached Poetry setup + id: cached-poetry + uses: actions/cache@v5 + with: + path: ~/.local + key: poetry-${{ matrix.os }}-2 # increment to reset cache - name: Setup Poetry + if: steps.cached-poetry.outputs.cache-hit != 'true' uses: snok/install-poetry@v1 + with: + version: 2.2.1 - - name: Install dependencies - run: poetry install - - - name: Build executable - run: poetry run pyinstaller pyinstaller.spec + - name: Add Poetry to PATH + run: echo "$HOME/.local/bin" >> $GITHUB_PATH - - name: Test executable - run: ./dist/cycode --version + - name: Install dependencies + run: poetry install --without dev,test - - name: Sign macOS executable - if: ${{ startsWith(matrix.os, 'macos') }} + - name: Import macOS signing certificate + if: runner.os == 'macOS' env: APPLE_CERT: ${{ secrets.APPLE_CERT }} APPLE_CERT_PWD: ${{ secrets.APPLE_CERT_PWD }} @@ -75,11 +107,58 @@ jobs: security import $CERTIFICATE_PATH -P "$APPLE_CERT_PWD" -A -t cert -f pkcs12 -k $KEYCHAIN_PATH security list-keychain -d user -s $KEYCHAIN_PATH - # sign executable - codesign --deep --force --options=runtime --entitlements entitlements.plist --sign "$APPLE_CERT_NAME" --timestamp dist/cycode + - name: Build executable (onefile) + if: matrix.mode == 'onefile' + env: + APPLE_CERT_NAME: ${{ secrets.APPLE_CERT_NAME }} + run: | + poetry run pyinstaller pyinstaller.spec + echo "PATH_TO_CYCODE_CLI_EXECUTABLE=dist/cycode-cli" >> $GITHUB_ENV + + - name: Build executable (onedir) + if: matrix.mode == 'onedir' + env: + CYCODE_ONEDIR_MODE: 1 + APPLE_CERT_NAME: ${{ secrets.APPLE_CERT_NAME }} + run: | + poetry run pyinstaller pyinstaller.spec + echo "PATH_TO_CYCODE_CLI_EXECUTABLE=dist/cycode-cli/cycode-cli" >> $GITHUB_ENV + + - name: Test executable + run: time $PATH_TO_CYCODE_CLI_EXECUTABLE status + + - name: Codesign onedir binaries + if: runner.os == 'macOS' && matrix.mode == 'onedir' + env: + APPLE_CERT_NAME: ${{ secrets.APPLE_CERT_NAME }} + run: | + # The standalone _internal/Python fails codesign --verify --strict because it was + # extracted from Python.framework without Info.plist context. + # Fix: remove the bare copy and replace with the framework version's binary, + # then delete the framework directory (it's redundant). + if [ -d dist/cycode-cli/_internal/Python.framework ]; then + FRAMEWORK_PYTHON=$(find dist/cycode-cli/_internal/Python.framework/Versions -name "Python" -type f | head -1) + if [ -n "$FRAMEWORK_PYTHON" ]; then + echo "Replacing _internal/Python with framework binary" + rm dist/cycode-cli/_internal/Python + cp "$FRAMEWORK_PYTHON" dist/cycode-cli/_internal/Python + fi + rm -rf dist/cycode-cli/_internal/Python.framework + fi + + # Sign all Mach-O binaries (excluding the main executable) + while IFS= read -r file; do + if file -b "$file" | grep -q "Mach-O"; then + echo "Signing: $file" + codesign --force --sign "$APPLE_CERT_NAME" --timestamp --options runtime "$file" + fi + done < <(find dist/cycode-cli -type f ! -name "cycode-cli") + + # Re-sign the main executable with entitlements (must be last) + codesign --force --sign "$APPLE_CERT_NAME" --timestamp --options runtime --entitlements entitlements.plist dist/cycode-cli/cycode-cli - name: Notarize macOS executable - if: ${{ startsWith(matrix.os, 'macos') }} + if: runner.os == 'macOS' env: APPLE_NOTARIZATION_EMAIL: ${{ secrets.APPLE_NOTARIZATION_EMAIL }} APPLE_NOTARIZATION_PWD: ${{ secrets.APPLE_NOTARIZATION_PWD }} @@ -88,20 +167,155 @@ jobs: # create keychain profile xcrun notarytool store-credentials "notarytool-profile" --apple-id "$APPLE_NOTARIZATION_EMAIL" --team-id "$APPLE_NOTARIZATION_TEAM_ID" --password "$APPLE_NOTARIZATION_PWD" - # create zip file (notarization does not support binaries) - ditto -c -k --keepParent dist/cycode notarization.zip + # create zip file (notarization does not support bare binaries) + ditto -c -k --keepParent dist/cycode-cli notarization.zip # notarize app (this will take a while) - xcrun notarytool submit notarization.zip --keychain-profile "notarytool-profile" --wait + NOTARIZE_OUTPUT=$(xcrun notarytool submit notarization.zip --keychain-profile "notarytool-profile" --wait 2>&1) || true + echo "$NOTARIZE_OUTPUT" + + # extract submission ID for log retrieval + SUBMISSION_ID=$(echo "$NOTARIZE_OUTPUT" | grep " id:" | head -1 | awk '{print $2}') + + # check notarization status explicitly + if echo "$NOTARIZE_OUTPUT" | grep -q "status: Accepted"; then + echo "Notarization succeeded!" + else + echo "Notarization failed! Fetching log for details..." + if [ -n "$SUBMISSION_ID" ]; then + xcrun notarytool log "$SUBMISSION_ID" --keychain-profile "notarytool-profile" || true + fi + exit 1 + fi - # we can't staple the app because it's executable. we should only staple app bundles like .dmg - # xcrun stapler staple dist/cycode + # we can't staple the app because it's executable - - name: Test signed executable - if: ${{ startsWith(matrix.os, 'macos') }} - run: ./dist/cycode --version + - name: Verify macOS code signatures + if: runner.os == 'macOS' + run: | + FAILED=false + while IFS= read -r file; do + if file -b "$file" | grep -q "Mach-O"; then + if ! codesign --verify "$file" 2>&1; then + echo "INVALID: $file" + codesign -dv "$file" 2>&1 || true + FAILED=true + else + echo "OK: $file" + fi + fi + done < <(find dist/cycode-cli -type f) + + if [ "$FAILED" = true ]; then + echo "Found binaries with invalid signatures!" + exit 1 + fi + + codesign -dv --verbose=4 $PATH_TO_CYCODE_CLI_EXECUTABLE + + - name: Test macOS signed executable + if: runner.os == 'macOS' + run: | + file -b $PATH_TO_CYCODE_CLI_EXECUTABLE + time $PATH_TO_CYCODE_CLI_EXECUTABLE status + + - name: Import cert for Windows and setup envs + if: runner.os == 'Windows' + env: + SM_CLIENT_CERT_FILE_B64: ${{ secrets.SM_CLIENT_CERT_FILE_B64 }} + run: | + # import certificate + echo "$SM_CLIENT_CERT_FILE_B64" | base64 --decode > /d/Certificate_pkcs12.p12 + echo "SM_CLIENT_CERT_FILE=D:\\Certificate_pkcs12.p12" >> "$GITHUB_ENV" + + # add required soft to the path + echo "C:\Program Files (x86)\Windows Kits\10\App Certification Kit" >> $GITHUB_PATH + echo "C:\Program Files\DigiCert\DigiCert One Signing Manager Tools" >> $GITHUB_PATH + + - name: Sign Windows executable + if: runner.os == 'Windows' + shell: cmd + env: + SM_HOST: ${{ secrets.SM_HOST }} + SM_KEYPAIR_ALIAS: ${{ secrets.SM_KEYPAIR_ALIAS }} + SM_API_KEY: ${{ secrets.SM_API_KEY }} + SM_CLIENT_CERT_PASSWORD: ${{ secrets.SM_CLIENT_CERT_PASSWORD }} + SM_CODE_SIGNING_CERT_SHA1_HASH: ${{ secrets.SM_CODE_SIGNING_CERT_SHA1_HASH }} + run: | + :: setup SSM KSP + curl -X GET https://one.digicert.com/signingmanager/api-ui/v1/releases/smtools-windows-x64.msi/download -H "x-api-key:%SM_API_KEY%" -o smtools-windows-x64.msi + msiexec /i smtools-windows-x64.msi /quiet /qn + C:\Windows\System32\certutil.exe -csp "DigiCert Signing Manager KSP" -key -user + smctl windows certsync --keypair-alias=%SM_KEYPAIR_ALIAS% + + :: sign executable + signtool.exe sign /sha1 %SM_CODE_SIGNING_CERT_SHA1_HASH% /tr http://timestamp.digicert.com /td SHA256 /fd SHA256 ".\dist\cycode-cli.exe" - - uses: actions/upload-artifact@v3 + - name: Test Windows signed executable + if: runner.os == 'Windows' + shell: cmd + run: | + :: call executable and expect correct output + .\dist\cycode-cli.exe status + + :: verify signature + signtool.exe verify /v /pa ".\dist\cycode-cli.exe" + + - name: Prepare files for artifact and release (rename and calculate sha256) + run: echo "ARTIFACT_NAME=$(./process_executable_file.py dist/cycode-cli)" >> $GITHUB_ENV + + - name: Upload files as artifact + uses: actions/upload-artifact@v4 with: - name: cycode-cli-${{ matrix.os }} + name: ${{ env.ARTIFACT_NAME }} path: dist + + - name: Verify macOS artifact end-to-end + if: runner.os == 'macOS' && matrix.mode == 'onedir' + uses: actions/download-artifact@v8 + with: + name: ${{ env.ARTIFACT_NAME }} + path: /tmp/artifact-verify + + - name: Verify macOS artifact signatures and run with quarantine + if: runner.os == 'macOS' && matrix.mode == 'onedir' + run: | + # extract the onedir zip exactly as an end user would + ARCHIVE=$(find /tmp/artifact-verify -name "*.zip" | head -1) + echo "Verifying archive: $ARCHIVE" + unzip "$ARCHIVE" -d /tmp/artifact-extracted + + # verify all Mach-O code signatures + FAILED=false + while IFS= read -r file; do + if file -b "$file" | grep -q "Mach-O"; then + if ! codesign --verify "$file" 2>&1; then + echo "INVALID: $file" + codesign -dv "$file" 2>&1 || true + FAILED=true + else + echo "OK: $file" + fi + fi + done < <(find /tmp/artifact-extracted -type f) + + if [ "$FAILED" = true ]; then + echo "Artifact contains binaries with invalid signatures!" + exit 1 + fi + + # simulate download quarantine and test execution + # this is the definitive test — it triggers the same dlopen checks end users experience + find /tmp/artifact-extracted -type f -exec xattr -w com.apple.quarantine "0081;$(printf '%x' $(date +%s));CI;$(uuidgen)" {} \; + EXECUTABLE=$(find /tmp/artifact-extracted -name "cycode-cli" -type f | head -1) + echo "Testing quarantined executable: $EXECUTABLE" + time "$EXECUTABLE" status + + - name: Upload files to release + if: ${{ github.event_name == 'workflow_dispatch' && inputs.publish }} + uses: svenstaro/upload-release-action@v2 + with: + file: dist/* + tag: ${{ env.LATEST_TAG }} + overwrite: true + file_glob: true diff --git a/.github/workflows/docker-image-dev.yml b/.github/workflows/docker-image-dev.yml deleted file mode 100644 index b0743afc..00000000 --- a/.github/workflows/docker-image-dev.yml +++ /dev/null @@ -1,38 +0,0 @@ -on: - push: - branches: dev - -permissions: - contents: read - -jobs: - main: - runs-on: ubuntu-latest - steps: - - - name: Checkout - uses: actions/checkout@v2 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - - name: Login to DockerHub Registry - env: - DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} - DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} - run: echo "$DOCKERHUB_PASSWORD" | docker login -u "$DOCKERHUB_USER" --password-stdin - - - name: Build and push - id: docker_build - uses: docker/build-push-action@v3 - with: - context: . - file: ./Dockerfile - push: true - tags: cycodehq/cycode_cli:dev - - - name: Image digest - run: echo ${{ steps.docker_build.outputs.digest }} diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 8912d969..4e2d4ee8 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -1,46 +1,91 @@ +name: Build Docker Image. On tag creation push to Docker Hub. On dispatch event build the latest tag and push to Docker Hub + on: workflow_dispatch: - -permissions: - # Write permission needed for creating a tag. - contents: write + pull_request: + push: + tags: [ 'v*.*.*' ] jobs: - main: + docker: runs-on: ubuntu-latest + steps: - - - name: Checkout - uses: actions/checkout@v2 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - - name: Login to DockerHub Registry - env: - DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} - DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} - run: echo "$DOCKERHUB_PASSWORD" | docker login -u "$DOCKERHUB_USER" --password-stdin - - - name: Bump version - id: bump_version - uses: anothrNick/github-tag-action@1.36.0 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - DEFAULT_BUMP: minor - - - - name: Build and push + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Get latest release tag + id: latest_tag + run: | + LATEST_TAG=$(git describe --tags `git rev-list --tags --max-count=1`) + echo "LATEST_TAG=$LATEST_TAG" >> $GITHUB_OUTPUT + + - name: Check out latest release tag + if: ${{ github.event_name == 'workflow_dispatch' }} + run: | + git checkout ${{ steps.latest_tag.outputs.LATEST_TAG }} + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: '3.9' + + - name: Load cached Poetry setup + id: cached_poetry + uses: actions/cache@v5 + with: + path: ~/.local + key: poetry-ubuntu-1 # increment to reset cache + + - name: Setup Poetry + if: steps.cached_poetry.outputs.cache-hit != 'true' + uses: snok/install-poetry@v1 + with: + version: 2.2.1 + + - name: Add Poetry to PATH + run: echo "$HOME/.local/bin" >> $GITHUB_PATH + + - name: Install Poetry Plugin + run: poetry self add "poetry-dynamic-versioning[plugin]" + + - name: Get CLI Version + id: cli_version + run: | + echo "::debug::Package version: $(poetry version --short)" + echo "CLI_VERSION=$(poetry version --short)" >> $GITHUB_OUTPUT + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Login to Docker Hub + if: ${{ github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/tags/v') }} + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USER }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + - name: Build and push id: docker_build - uses: docker/build-push-action@v3 + if: ${{ github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/tags/v') }} + uses: docker/build-push-action@v7 with: context: . - file: ./Dockerfile + platforms: linux/amd64,linux/arm64 push: true - tags: cycodehq/cycode_cli:${{ steps.bump_version.outputs.new_tag }} - - - name: Image digest - run: echo ${{ steps.docker_build.outputs.digest }} + tags: cycodehq/cycode_cli:${{ steps.latest_tag.outputs.LATEST_TAG }},cycodehq/cycode_cli:latest + + - name: Verify build + id: docker_verify_build + if: ${{ github.event_name != 'workflow_dispatch' && !startsWith(github.ref, 'refs/tags/v') }} + uses: docker/build-push-action@v7 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: false + tags: cycodehq/cycode_cli:${{ steps.cli_version.outputs.CLI_VERSION }} diff --git a/.github/workflows/pre_release.yml b/.github/workflows/pre_release.yml index 13ae474b..802f4e27 100644 --- a/.github/workflows/pre_release.yml +++ b/.github/workflows/pre_release.yml @@ -25,19 +25,33 @@ jobs: install.python-poetry.org pypi.org upload.pypi.org + *.sigstore.dev - name: Checkout repository uses: actions/checkout@v3 with: fetch-depth: 0 - - name: Set up Python 3.7 - uses: actions/setup-python@v4 + - name: Set up Python + uses: actions/setup-python@v6 with: - python-version: '3.7' + python-version: '3.9' + + - name: Load cached Poetry setup + id: cached-poetry + uses: actions/cache@v5 + with: + path: ~/.local + key: poetry-ubuntu-1 # increment to reset cache - name: Setup Poetry + if: steps.cached-poetry.outputs.cache-hit != 'true' uses: snok/install-poetry@v1 + with: + version: 2.2.1 + + - name: Add Poetry to PATH + run: echo "$HOME/.local/bin" >> $GITHUB_PATH - name: Install Poetry Plugin run: poetry self add "poetry-dynamic-versioning[plugin]" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 70a60c33..88f86ef7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -24,19 +24,33 @@ jobs: install.python-poetry.org pypi.org upload.pypi.org + *.sigstore.dev - name: Checkout repository uses: actions/checkout@v3 with: fetch-depth: 0 - - name: Set up Python 3.7 - uses: actions/setup-python@v4 + - name: Set up Python + uses: actions/setup-python@v6 with: - python-version: '3.7' + python-version: '3.9' + + - name: Load cached Poetry setup + id: cached-poetry + uses: actions/cache@v5 + with: + path: ~/.local + key: poetry-ubuntu-1 # increment to reset cache - name: Setup Poetry + if: steps.cached-poetry.outputs.cache-hit != 'true' uses: snok/install-poetry@v1 + with: + version: 2.2.1 + + - name: Add Poetry to PATH + run: echo "$HOME/.local/bin" >> $GITHUB_PATH - name: Install Poetry Plugin run: poetry self add "poetry-dynamic-versioning[plugin]" diff --git a/.github/workflows/ruff.yml b/.github/workflows/ruff.yml new file mode 100644 index 00000000..ae6c7913 --- /dev/null +++ b/.github/workflows/ruff.yml @@ -0,0 +1,51 @@ +name: Ruff (linter and code formatter) + +on: [ pull_request, push ] + +jobs: + ruff: + runs-on: ubuntu-latest + steps: + - name: Run Cimon + uses: cycodelabs/cimon-action@v0 + with: + client-id: ${{ secrets.CIMON_CLIENT_ID }} + secret: ${{ secrets.CIMON_SECRET }} + prevent: true + allowed-hosts: > + files.pythonhosted.org + install.python-poetry.org + pypi.org + + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: 3.9 + + - name: Load cached Poetry setup + id: cached-poetry + uses: actions/cache@v5 + with: + path: ~/.local + key: poetry-ubuntu-1 # increment to reset cache + + - name: Setup Poetry + if: steps.cached-poetry.outputs.cache-hit != 'true' + uses: snok/install-poetry@v1 + with: + version: 2.2.1 + + - name: Add Poetry to PATH + run: echo "$HOME/.local/bin" >> $GITHUB_PATH + + - name: Install dependencies + run: poetry install + + - name: Run linter check + run: poetry run ruff check --output-format=github . + + - name: Run code style check + run: poetry run ruff format --check . diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 27450b71..c69fe4ac 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -20,20 +20,34 @@ jobs: files.pythonhosted.org install.python-poetry.org pypi.org + *.ingest.us.sentry.io - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v6 with: - python-version: '3.7' + python-version: '3.9' + + - name: Load cached Poetry setup + id: cached-poetry + uses: actions/cache@v5 + with: + path: ~/.local + key: poetry-ubuntu-1 # increment to reset cache - name: Setup Poetry + if: steps.cached-poetry.outputs.cache-hit != 'true' uses: snok/install-poetry@v1 + with: + version: 2.2.1 + + - name: Add Poetry to PATH + run: echo "$HOME/.local/bin" >> $GITHUB_PATH - name: Install dependencies run: poetry install - name: Run Tests - run: poetry run pytest + run: poetry run python -m pytest diff --git a/.github/workflows/tests_full.yml b/.github/workflows/tests_full.yml index bdcc49d4..65426b13 100644 --- a/.github/workflows/tests_full.yml +++ b/.github/workflows/tests_full.yml @@ -13,7 +13,7 @@ jobs: strategy: matrix: os: [ macos-latest, ubuntu-latest, windows-latest ] - python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11" ] + python-version: [ "3.9", "3.10", "3.11", "3.12", "3.13", "3.14" ] runs-on: ${{matrix.os}} @@ -33,20 +33,43 @@ jobs: files.pythonhosted.org install.python-poetry.org pypi.org + *.ingest.us.sentry.io - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 + with: + fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} + - name: Load cached Poetry setup + id: cached-poetry + uses: actions/cache@v5 + with: + path: ~/.local + key: poetry-${{ matrix.os }}-${{ matrix.python-version }}-3 # increment to reset cache + - name: Setup Poetry + if: steps.cached-poetry.outputs.cache-hit != 'true' uses: snok/install-poetry@v1 + with: + version: 2.2.1 + + - name: Add Poetry to PATH + run: echo "$HOME/.local/bin" >> $GITHUB_PATH - name: Install dependencies run: poetry install - - name: Run Tests - run: poetry run pytest + - name: Run executable test + # we care about the one Python version that will be used to build the executable + if: matrix.python-version == '3.13' + run: | + poetry run pyinstaller pyinstaller.spec + ./dist/cycode-cli version + + - name: Run pytest + run: poetry run python -m pytest diff --git a/.gitignore b/.gitignore index 1de92d03..7f27180f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ +.DS_Store .idea *.iml .env +.ruff_cache/ # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml index d3b55ce6..fd2bfbed 100644 --- a/.pre-commit-hooks.yaml +++ b/.pre-commit-hooks.yaml @@ -1,10 +1,39 @@ - id: cycode - name: Cycode pre commit defender + name: Cycode Secrets pre-commit defender language: python + language_version: python3 entry: cycode - args: [ 'scan', 'pre_commit' ] + args: [ '-o', 'text', '--no-progress-meter', 'scan', '-t', 'secret', 'pre-commit' ] - id: cycode-sca - name: Cycode SCA pre commit defender + name: Cycode SCA pre-commit defender language: python + language_version: python3 entry: cycode - args: [ 'scan', '--scan-type', 'sca', 'pre_commit' ] \ No newline at end of file + args: [ '-o', 'text', '--no-progress-meter', 'scan', '-t', 'sca', 'pre-commit' ] +- id: cycode-sast + name: Cycode SAST pre-commit defender + language: python + language_version: python3 + entry: cycode + args: [ '-o', 'text', '--no-progress-meter', 'scan', '-t', 'sast', 'pre-commit' ] +- id: cycode-pre-push + name: Cycode Secrets pre-push defender + language: python + language_version: python3 + entry: cycode + args: [ '-o', 'text', '--no-progress-meter', 'scan', '-t', 'secret', 'pre-push' ] + stages: [pre-push] +- id: cycode-sca-pre-push + name: Cycode SCA pre-push defender + language: python + language_version: python3 + entry: cycode + args: [ '-o', 'text', '--no-progress-meter', 'scan', '-t', 'sca', 'pre-push' ] + stages: [pre-push] +- id: cycode-sast-pre-push + name: Cycode SAST pre-push defender + language: python + language_version: python3 + entry: cycode + args: [ '-o', 'text', '--no-progress-meter', 'scan', '-t', 'sast', 'pre-push' ] + stages: [pre-push] diff --git a/CODEOWNERS b/CODEOWNERS index 874040fd..f05ffdb9 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1 +1 @@ -* @MarshalX @MichalBor @MaorDavidzon @artem-fedorov +* @elsapet @gotbadger @mateusz-sterczewski diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..857a27cd --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,165 @@ +![GitHub Actions Workflow Status](https://img.shields.io/github/actions/workflow/status/cycodehq/cycode-cli/tests.yml) +![PyPI - Version](https://img.shields.io/pypi/v/cycode) +![GitHub License](https://img.shields.io/github/license/cycodehq/cycode-cli) + +## How to contribute to Cycode CLI + +The minimum version of Python that we support is 3.9. +We recommend using this version for local development. +But it’s fine to use a higher version without using new features from these versions. + +The project is under Poetry project management. +To deal with it, you should install it on your system: + +Install Poetry (feel free to use Brew, etc.): + +```shell +curl -sSL https://install.python-poetry.org | python - -y +``` + +Add Poetry to PATH if required. + +Add a plugin to support dynamic versioning from Git Tags: + +```shell +poetry self add "poetry-dynamic-versioning[plugin]" +``` + +Install dependencies of the project: + +```shell +poetry install +``` + +Check that the version is valid (not 0.0.0): + +```shell +poetry version +``` + +You are ready to write code! + +To run the project use: + +```shell +poetry run cycode +``` + +or main entry point in an activated virtual environment: + +```shell +python cycode/cli/main.py +``` + +### Code linting and formatting + +We use `ruff`. +It is configured well, so you don’t need to do anything. +You can see all enabled rules in the `pyproject.toml` file. +Both tests and the main codebase are checked. +Try to avoid type annotations like `Any`, etc. + +GitHub Actions will check that your code is formatted well. You can run it locally: + +```shell +# lint +poetry run ruff check . +# format +poetry run ruff format . +``` + +Many rules support auto-fixing. You can run it with the `--fix` flag. + +Plugin for JB IDEs with auto formatting on save is available [here](https://plugins.jetbrains.com/plugin/20574-ruff). + +### Branching and versioning + +We use the `main` branch as the main one. +All development should be done in feature branches. +When you are ready create a Pull Request to the `main` branch. + +Each commit in the `main` branch will be built and published to PyPI as a pre-release! +Such builds could be installed with the `--pre` flag. For example: + +```shell +pip install --pre cycode +``` + +Also, you can select a specific version of the pre-release: + +```shell +pip install cycode==1.7.2.dev6 +``` + +We are using [Semantic Versioning](https://semver.org/) and the version is generated automatically from Git Tags. So, +when you are ready to release a new version, you should create a new Git Tag. The version will be generated from it. + +Pre-release versions are generated on distance from the latest Git Tag. For example, if the latest Git Tag is `1.7.2`, +then the next pre-release version will be `1.7.2.dev1`. + +We are using GitHub Releases to create Git Tags with changelogs. +For changelogs, we are using a standard template +of [Automatically generated release notes](https://docs.github.com/en/repositories/releasing-projects-on-github/automatically-generated-release-notes). + +### Testing + +We are using `pytest` for testing. You can run tests with: + +```shell +poetry run pytest +``` + +The library used for sending requests is [requests](https://github.com/psf/requests). +To mock requests, we are using the [responses](https://github.com/getsentry/responses) library. +All requests must be mocked. + +To see the code coverage of the project, you can run: + +```shell +poetry run coverage run -m pytest . +``` + +To generate the HTML report, you can run: + +```shell +poetry run coverage html +``` + +The report will be generated in the `htmlcov` folder. + +### Documentation + +Keep [README.md](README.md) up to date. +All CLI commands are documented automatically if you add a docstring to the command. +Clean up the changelog before release. + +### Publishing + +New versions are published automatically on the new GitHub Release. +It uses the OpenID Connect publishing mechanism to upload on PyPI. + +[Homebrew formula](https://formulae.brew.sh/formula/cycode) is updated automatically on the new PyPI release. + +The CLI is also distributed as executable files for Linux, macOS, and Windows. +It is powered by [PyInstaller](https://pyinstaller.org/) and the process is automated by GitHub Actions. +These executables are attached to GitHub Releases as assets. + +To pack the project locally, you should run: + +```shell +poetry build +``` + +It will create a `dist` folder with the package (sdist and wheel). You can install it locally: + +```shell +pip install dist/cycode-{version}-py3-none-any.whl +``` + +To create an executable file locally, you should run: + +```shell +poetry run pyinstaller pyinstaller.spec +``` + +It will create an executable file for **the current platform** in the `dist` folder. diff --git a/Dockerfile b/Dockerfile index a2197817..40d6fad3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,12 @@ -FROM python:3.8.16-alpine3.17 as base +FROM python:3.12.9-alpine3.21 AS base WORKDIR /usr/cycode/app -RUN apk add git=2.38.5-r0 +RUN apk add git=2.47.3-r0 -FROM base as builder -ENV POETRY_VERSION=1.4.2 +FROM base AS builder +ENV POETRY_VERSION=2.2.1 # deps are required to build cffi -RUN apk add --no-cache --virtual .build-deps gcc=12.2.1_git20220924-r4 libffi-dev=3.4.4-r0 musl-dev=1.2.3-r4 && \ +RUN apk add --no-cache --virtual .build-deps gcc=14.2.0-r4 libffi-dev=3.4.7-r0 musl-dev=1.2.5-r9 && \ pip install --no-cache-dir "poetry==$POETRY_VERSION" "poetry-dynamic-versioning[plugin]" && \ apk del .build-deps gcc libffi-dev musl-dev @@ -19,7 +19,7 @@ RUN poetry config virtualenvs.in-project true && \ poetry --no-cache install --only=main --no-root && \ poetry build -FROM base as final +FROM base AS final COPY --from=builder /usr/cycode/app/dist ./ RUN pip install --no-cache-dir cycode*.whl diff --git a/LICENCE b/LICENCE index 404e3d08..8d72be9a 100644 --- a/LICENCE +++ b/LICENCE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2022 cycodehq-public +Copyright (c) 2022 Cycode Ltd. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index 370001e7..b512c813 100644 --- a/README.md +++ b/README.md @@ -1,61 +1,77 @@ # Cycode CLI User Guide -The Cycode Command Line Interface (CLI) is an application you can install on your local machine which can scan your locally stored repositories for any secrets or infrastructure as code misconfigurations. +The Cycode Command Line Interface (CLI) is an application you can install locally to scan your repositories for secrets, infrastructure as code misconfigurations, software composition analysis vulnerabilities, and static application security testing issues. -This guide will guide you through both installation and usage. +This guide walks you through both installation and usage. # Table of Contents 1. [Prerequisites](#prerequisites) 2. [Installation](#installation) 1. [Install Cycode CLI](#install-cycode-cli) - 1. [Use `auth` command](#use-auth-command) - 2. [Use `configure` command](#use-configure-command) + 1. [Using the Auth Command](#using-the-auth-command) + 2. [Using the Configure Command](#using-the-configure-command) 3. [Add to Environment Variables](#add-to-environment-variables) 1. [On Unix/Linux](#on-unixlinux) 2. [On Windows](#on-windows) 2. [Install Pre-Commit Hook](#install-pre-commit-hook) -3. [Cycode Command](#cycode-command) -4. [Running a Scan](#running-a-scan) - 1. [Repository Scan](#repository-scan) - 1. [Branch Option](#branch-option) - 2. [Monitor Option](#monitor-option) - 3. [Report Option](#report-option) - 4. [Package Vulnerabilities Scan](#package-vulnerabilities-option) - 1. [License Compliance Option](#license-compliance-option) - 2. [Severity Threshold](#severity-threshold) - 5. [Path Scan](#path-scan) - 6. [Commit History Scan](#commit-history-scan) - 1. [Commit Range Option](#commit-range-option) - 7. [Pre-Commit Scan](#pre-commit-scan) -5. [Scan Results](#scan-results) - 1. [Show/Hide Secrets](#showhide-secrets) - 2. [Soft Fail](#soft-fail) - 3. [Example Scan Results](#example-scan-results) - 1. [Secrets Result Example](#secrets-result-example) - 2. [IaC Result Example](#iac-result-example) - 3. [SCA Result Example](#sca-result-example) - 4. [SAST Result Example](#sast-result-example) -6. [Ignoring Scan Results](#ignoring-scan-results) - 1. [Ignoring a Secret Value](#ignoring-a-secret-value) - 2. [Ignoring a Secret SHA Value](#ignoring-a-secret-sha-value) - 3. [Ignoring a Path](#ignoring-a-path) - 4. [Ignoring a Secret, IaC, or SCA Rule](#ignoring-a-secret-iac-sca-or-sast-rule) - 5. [Ignoring a Package](#ignoring-a-package) -7. [Syntax Help](#syntax-help) +3. [Cycode CLI Commands](#cycode-cli-commands) +4. [MCP Command](#mcp-command-experiment) + 1. [Starting the MCP Server](#starting-the-mcp-server) + 2. [Available Options](#available-options) + 3. [MCP Tools](#mcp-tools) + 4. [Usage Examples](#usage-examples) +5. [Scan Command](#scan-command) + 1. [Running a Scan](#running-a-scan) + 1. [Options](#options) + 1. [Severity Threshold](#severity-option) + 2. [Monitor](#monitor-option) + 3. [Cycode Report](#cycode-report-option) + 4. [Package Vulnerabilities](#package-vulnerabilities-option) + 5. [License Compliance](#license-compliance-option) + 6. [Lock Restore](#lock-restore-option) + 2. [Repository Scan](#repository-scan) + 1. [Branch Option](#branch-option) + 3. [Path Scan](#path-scan) + 1. [Terraform Plan Scan](#terraform-plan-scan) + 4. [Commit History Scan](#commit-history-scan) + 1. [Commit Range Option (Diff Scanning)](#commit-range-option-diff-scanning) + 5. [Pre-Commit Scan](#pre-commit-scan) + 6. [Pre-Push Scan](#pre-push-scan) + 2. [Scan Results](#scan-results) + 1. [Show/Hide Secrets](#showhide-secrets) + 2. [Soft Fail](#soft-fail) + 3. [Example Scan Results](#example-scan-results) + 1. [Secrets Result Example](#secrets-result-example) + 2. [IaC Result Example](#iac-result-example) + 3. [SCA Result Example](#sca-result-example) + 4. [SAST Result Example](#sast-result-example) + 4. [Company Custom Remediation Guidelines](#company-custom-remediation-guidelines) + 3. [Ignoring Scan Results](#ignoring-scan-results) + 1. [Ignoring a Secret Value](#ignoring-a-secret-value) + 2. [Ignoring a Secret SHA Value](#ignoring-a-secret-sha-value) + 3. [Ignoring a Path](#ignoring-a-path) + 4. [Ignoring a Secret, IaC, or SCA Rule](#ignoring-a-secret-iac-sca-or-sast-rule) + 5. [Ignoring a Package](#ignoring-a-package) + 6. [Ignoring via a config file](#ignoring-via-a-config-file) +6. [Report command](#report-command) + 1. [Generating SBOM Report](#generating-sbom-report) +7. [Import command](#import-command) +8. [Scan logs](#scan-logs) +9. [Syntax Help](#syntax-help) # Prerequisites -- The Cycode CLI application requires Python version 3.7 or later. -- Use the [`cycode auth` command](#use-auth-command) to authenticate to Cycode with the CLI - - Alternatively, a Cycode Client ID and Client Secret Key can be acquired using the steps from the [Service Account Token](https://docs.cycode.com/reference/creating-a-service-account-access-token) and [Personal Access Token](https://docs.cycode.com/reference/creating-a-personal-access-token-1) pages for details on obtaining these values. +- The Cycode CLI application requires Python version 3.9 or later. The MCP command is available only for Python 3.10 and above. If you're using an earlier Python version, this command will not be available. +- Use the [`cycode auth` command](#using-the-auth-command) to authenticate to Cycode with the CLI + - Alternatively, you can get a Cycode Client ID and Client Secret Key by following the steps detailed in the [Service Account Token](https://docs.cycode.com/docs/en/service-accounts) and [Personal Access Token](https://docs.cycode.com/v1/docs/managing-personal-access-tokens) pages, which contain details on getting these values. # Installation -The following installation steps are applicable on both Windows and UNIX / Linux operating systems. +The following installation steps are applicable to both Windows and UNIX / Linux operating systems. -> :memo: **Note**
-> The following steps assume the use of `python3` and `pip3` for Python-related commands, but some systems may instead use the `python` and `pip` commands, depending on your Python environment’s configuration. +> [!NOTE] +> The following steps assume the use of `python3` and `pip3` for Python-related commands; however, some systems may instead use the `python` and `pip` commands, depending on your Python environment’s configuration. ## Install Cycode CLI @@ -63,21 +79,37 @@ To install the Cycode CLI application on your local machine, perform the followi 1. Open your command line or terminal application. -2. Execute the following command: +2. Execute one of the following commands: - `pip3 install cycode` + - To install from [PyPI](https://pypi.org/project/cycode/): -3. Navigate to the top directory of the local repository you wish to scan. + ```bash + pip3 install cycode + ``` -4. There are three methods to set the Cycode client ID and client secret: + - To install from [Homebrew](https://formulae.brew.sh/formula/cycode): - - [cycode auth](#use-auth-command) (**Recommended**) - - [cycode configure](#use-configure-command) + ```bash + brew install cycode + ``` + + - To install from [GitHub Releases](https://github.com/cycodehq/cycode-cli/releases) navigate and download executable for your operating system and architecture, then run the following command: + + ```bash + cd /path/to/downloaded/cycode-cli + chmod +x cycode + ./cycode + ``` + +3. Finally authenticate the CLI. There are three methods to set the Cycode client ID and credentials (client secret or OIDC ID token): + + - [cycode auth](#using-the-auth-command) (**Recommended**) + - [cycode configure](#using-the-configure-command) - Add them to your [environment variables](#add-to-environment-variables) -### Use auth Command +### Using the Auth Command -> :memo: **Note**
+> [!NOTE] > This is the **recommended** method for setting up your local machine to authenticate with Cycode CLI. 1. Type the following command into your terminal/command line window: @@ -86,66 +118,70 @@ To install the Cycode CLI application on your local machine, perform the followi 2. A browser window will appear, asking you to log into Cycode (as seen below): -![Cycode login](https://raw.githubusercontent.com/cycodehq-public/cycode-cli/main/images/cycode_login.png) + Cycode login -3. Enter you login credentials on this page and log in. +3. Enter your login credentials on this page and log in. -4. You will eventually be taken to this page, where you will be asked to choose the business group you want to authorize Cycode with (if applicable): +4. You will eventually be taken to the page below, where you'll be asked to choose the business group you want to authorize Cycode with (if applicable): -![authorize CLI](https://raw.githubusercontent.com/cycodehq-public/cycode-cli/main/images/authorize_cli.png) + authorize CLI -> :memo: **Note**
-> This will be the default method for authenticating with the Cycode CLI. + > [!NOTE] + > This will be the default method for authenticating with the Cycode CLI. -5. Click the **Allow** button to authorize the Cycode CLI on the chosen business group. +5. Click the **Allow** button to authorize the Cycode CLI on the selected business group. -![allow CLI](https://raw.githubusercontent.com/cycodehq-public/cycode-cli/main/images/allow_cli.png) + allow CLI -6. Once done, you will see the following screen, if it was successfully selected: +6. Once completed, you'll see the following screen if it was selected successfully: -![successfully auth](https://raw.githubusercontent.com/cycodehq-public/cycode-cli/main/images/successfully_auth.png) + successfully auth 7. In the terminal/command line screen, you will see the following when exiting the browser window: - ```bash - Successfully logged into cycode - ``` + `Successfully logged into cycode` -### Use configure Command +### Using the Configure Command -> :memo: **Note**
-> If you already setup your Cycode client ID and client secret through the Linux or Windows environment variables, those credentials will take precedent over this method +> [!NOTE] +> If you already set up your Cycode Client ID and Client Secret through the Linux or Windows environment variables, those credentials will take precedent over this method. 1. Type the following command into your terminal/command line window: - `cycode configure` + ```bash + cycode configure + ``` - You will see the following appear: +2. Enter your Cycode API URL value (you can leave blank to use default value). - ```bash - Update credentials in file (/Users/travislloyd/.cycode/credentials.yaml) - cycode client id []: - ``` + `Cycode API URL [https://api.cycode.com]: https://api.onpremise.com` -2. Enter your Cycode client ID value. +3. Enter your Cycode APP URL value (you can leave blank to use default value). - ```bash - cycode client id []: 7fe5346b-xxxx-xxxx-xxxx-55157625c72d - ``` + `Cycode APP URL [https://app.cycode.com]: https://app.onpremise.com` -3. Enter your Cycode client secret value. +4. Enter your Cycode Client ID value. - ```bash - cycode client secret []: c1e24929-xxxx-xxxx-xxxx-8b08c1839a2e - ``` + `Cycode Client ID []: 7fe5346b-xxxx-xxxx-xxxx-55157625c72d` -4. If the values were entered successfully, you will see the following message: +5. Enter your Cycode Client Secret value (skip if you plan to use an OIDC ID token). - ```bash - Successfully configured CLI credentials! - ``` + `Cycode Client Secret []: c1e24929-xxxx-xxxx-xxxx-8b08c1839a2e` + +6. Enter your Cycode OIDC ID Token value (optional). -If you go into the `.cycode` folder under you user folder, you will find these credentials were created and placed in the `credentials.yaml` file in that folder. + `Cycode ID Token []: eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9...` + +7. If the values were entered successfully, you'll see the following message: + + `Successfully configured CLI credentials!` + + or/and + + `Successfully configured Cycode URLs!` + +If you go into the `.cycode` folder under your user folder, you'll find these credentials were created and placed in the `credentials.yaml` file in that folder. +The URLs were placed in the `config.yaml` file in that folder. ### Add to Environment Variables @@ -153,261 +189,540 @@ If you go into the `.cycode` folder under you user folder, you will find these c ```bash export CYCODE_CLIENT_ID={your Cycode ID} +``` + +and + +```bash export CYCODE_CLIENT_SECRET={your Cycode Secret Key} ``` +If your organization uses OIDC authentication, you can provide the ID token instead (or in addition): + +```bash +export CYCODE_ID_TOKEN={your Cycode OIDC ID token} +``` + #### On Windows 1. From the Control Panel, navigate to the System menu: -![](https://raw.githubusercontent.com/cycodehq-public/cycode-cli/main/images/image1.png) + system menu 2. Next, click Advanced system settings: -![](https://raw.githubusercontent.com/cycodehq-public/cycode-cli/main/images/image2.png) + advanced system setting 3. In the System Properties window that opens, click the Environment Variables button: -![](https://raw.githubusercontent.com/cycodehq-public/cycode-cli/main/images/image3.png) + environments variables button + +4. Create `CYCODE_CLIENT_ID` and `CYCODE_CLIENT_SECRET` variables with values matching your ID and Secret Key, respectively. If you authenticate via OIDC, add `CYCODE_ID_TOKEN` with your OIDC ID token value as well: -4. Create `CYCODE_CLIENT_ID` and `CYCODE_CLIENT_SECRET` variables with values matching your ID and Secret Key, respectively: + environment variables window -![](https://raw.githubusercontent.com/cycodehq-public/cycode-cli/main/images/image4.png) +5. Insert the `cycode.exe` into the path to complete the installation. ## Install Pre-Commit Hook -Cycode’s pre-commit hook can be set up within your local repository so that the Cycode CLI application will automatically identify any issues with your code before you commit it to your codebase. +Cycode's pre-commit and pre-push hooks can be set up within your local repository so that the Cycode CLI application will identify any issues with your code automatically before you commit or push it to your codebase. + +> [!NOTE] +> pre-commit and pre-push hooks are not available for IaC scans. Perform the following steps to install the pre-commit hook: -1. Install the pre-commit framework: +### Installing Pre-Commit Hook + +1. Install the pre-commit framework (Python 3.9 or higher must be installed): - `pip3 install pre-commit` + ```bash + pip3 install pre-commit + ``` -2. Navigate to the top directory of the local repository you wish to scan. +2. Navigate to the top directory of the local Git repository you wish to configure. 3. Create a new YAML file named `.pre-commit-config.yaml` (include the beginning `.`) in the repository’s top directory that contains the following: -```yaml -repos: - - repo: https://github.com/cycodehq-public/cycode-cli - rev: stable - hooks: - - id: cycode - language_version: python3 - stages: - - commit -``` + ```yaml + repos: + - repo: https://github.com/cycodehq/cycode-cli + rev: v3.5.0 + hooks: + - id: cycode + stages: [pre-commit] + ``` -4. Install Cycode’s hook: +4. Modify the created file for your specific needs. Use hook ID `cycode` to enable scan for Secrets. Use hook ID `cycode-sca` to enable SCA scan. Use hook ID `cycode-sast` to enable SAST scan. If you want to enable all scanning types, use this configuration: + + ```yaml + repos: + - repo: https://github.com/cycodehq/cycode-cli + rev: v3.5.0 + hooks: + - id: cycode + stages: [pre-commit] + - id: cycode-sca + stages: [pre-commit] + - id: cycode-sast + stages: [pre-commit] + ``` - `pre-commit install` +5. Install Cycode’s hook: -> :memo: **Note**
-> Successful hook installation will result in the message:
-`Pre-commit installed at .git/hooks/pre-commit` + ```bash + pre-commit install + ``` -# Cycode Command + A successful hook installation will result in the message: `Pre-commit installed at .git/hooks/pre-commit`. -The following are the options and commands available with the Cycode CLI application: +6. Keep the pre-commit hook up to date: -| Option | Description | -|--------------------------------|-------------------------------------------------------------------| -| `--output [text\|json\|table]` | Specify the output (`text`/`json`/`table`). The default is `text` | -| `-v`, `--verbose` | Show detailed logs | -| `--version` | Show the version and exit. | -| `--help` | Show options for given command. | + ```bash + pre-commit autoupdate + ``` -| Command | Description | -|-------------------------------------|-------------| -| [auth](#use-auth-command) | Authenticates your machine to associate CLI with your Cycode account. | -| [configure](#use-configure-command) | Initial command to authenticate your CLI client with Cycode using client ID and client secret. | -| [ignore](#ingoring-scan-results) | Ignore a specific value, path or rule ID | -| [scan](#running-a-scan) | Scan content for secrets/IaC/SCA/SAST violations. You need to specify which scan type: `ci`/`commit_history`/`path`/`repository`/etc | + It will automatically bump `rev` in `.pre-commit-config.yaml` to the latest available version of Cycode CLI. -# Running a Scan +> [!NOTE] +> Trigger happens on `git commit` command. +> Hook triggers only on the files that are staged for commit. -The Cycode CLI application offers several types of scans so that you can choose the option that best fits your case. The following are the current options and commands available: +### Installing Pre-Push Hook -| Option | Description | -|--------------------------------------|----------------------------------------------------------------------------| -| `-t, --scan-type [secret\|iac\|sca\|sast]` | Specify the scan you wish to execute (`secret`/`iac`/`sca`/`sast`), the default is `secret` | -| `--secret TEXT` | Specify a Cycode client secret for this specific scan execution | -| `--client-id TEXT` | Specify a Cycode client ID for this specific scan execution | -| `--show-secret BOOLEAN` | Show secrets in plain text. See [Show/Hide Secrets](#showhide-secrets) section for more details. | -| `--soft-fail BOOLEAN` | Run scan without failing, always return a non-error status code. See [Soft Fail](#soft-fail) section for more details. | -| `--severity-threshold [INFO\|LOW\|MEDIUM\|HIGH\|CRITICAL]` | Show only violations at the specified level or higher (supported for the SCA scan type only). | -| `--sca-scan` | Specify the SCA scan you wish to execute (`package-vulnerabilities`/`license-compliance`). The default is both | -| `--monitor` | When specified, the scan results will be recorded in the knowledge graph. Please note that when working in `monitor` mode, the knowledge graph will not be updated as a result of SCM events (Push, Repo creation). (Supported for SCA scan type only). | -| `--report` | When specified, a violations report will be generated. A URL link to the report will be printed as an output to the command execution | -| `--help` | Show options for given command. | - -| Command | Description | -|----------------------------------------|-----------------------------------------------------------------| -| [commit_history](#commit-history-scan) | Scan all the commits history in this git repository | -| [path](#path-scan) | Scan the files in the path supplied in the command | -| [pre_commit](#pre-commit-scan) | Use this command to scan the content that was not committed yet | -| [repository](#repository-scan) | Scan git repository including its history | - -## Repository Scan - -A repository scan examines an entire local repository for any exposed secrets or insecure misconfigurations. This more holistic scan type looks at everything: the current state of your repository and its commit history. It will look not only for currently exposed secrets within the repository but previously deleted secrets as well. +To install the pre-push hook in addition to or instead of the pre-commit hook: -To execute a full repository scan, execute the following: +1. Add the pre-push hooks to your `.pre-commit-config.yaml` file: -`cycode scan repository {{path}}` + ```yaml + repos: + - repo: https://github.com/cycodehq/cycode-cli + rev: v3.5.0 + hooks: + - id: cycode-pre-push + stages: [pre-push] + ``` -For example, consider a scenario in which you want to scan your repository stored in `~/home/git/codebase`. You could then execute the following: +2. Install the pre-push hook: -`cycode scan repository ~/home/git/codebase` + ```bash + pre-commit install --hook-type pre-push + ``` -The following option is available for use with this command: +3. For both pre-commit and pre-push hooks, use: -| Option | Description | -|---------------------|-------------| -| `-b, --branch TEXT` | Branch to scan, if not set scanning the default branch | + ```bash + pre-commit install + pre-commit install --hook-type pre-push + ``` -### Branch Option +> [!NOTE] +> Pre-push hooks trigger on `git push` command and scan only the commits about to be pushed. -To scan a specific branch of your local repository, add the argument `-b` (alternatively, `--branch`) followed by the name of the branch you wish to scan. +# Cycode CLI Commands -Consider the previous example. If you wanted to only scan a branch named `dev`, you could execute the following: +The following are the options and commands available with the Cycode CLI application: -`cycode scan repository ~/home/git/codebase -b dev` +| Option | Description | +|-------------------------------------------------------------------|------------------------------------------------------------------------------------| +| `-v`, `--verbose` | Show detailed logs. | +| `--no-progress-meter` | Do not show the progress meter. | +| `--no-update-notifier` | Do not check CLI for updates. | +| `-o`, `--output [rich\|text\|json\|table]` | Specify the output type. The default is `rich`. | +| `--client-id TEXT` | Specify a Cycode client ID for this specific scan execution. | +| `--client-secret TEXT` | Specify a Cycode client secret for this specific scan execution. | +| `--id-token TEXT` | Specify a Cycode OIDC ID token for this specific scan execution. | +| `--install-completion` | Install completion for the current shell.. | +| `--show-completion [bash\|zsh\|fish\|powershell\|pwsh]` | Show completion for the specified shell, to copy it or customize the installation. | +| `-h`, `--help` | Show options for given command. | -or: +| Command | Description | +|-------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------| +| [auth](#using-the-auth-command) | Authenticate your machine to associate the CLI with your Cycode account. | +| [configure](#using-the-configure-command) | Initial command to configure your CLI client authentication. | +| [ignore](#ignoring-scan-results) | Ignore a specific value, path or rule ID. | +| [mcp](#mcp-command-experiment) | Start the Model Context Protocol (MCP) server to enable AI integration with Cycode scanning capabilities. | +| [scan](#running-a-scan) | Scan the content for Secrets/IaC/SCA/SAST violations. You`ll need to specify which scan type to perform: commit-history/path/repository/etc. | +| [report](#report-command) | Generate report. You will need to specify which report type to perform as SBOM. | +| status | Show the CLI status and exit. | -`cycode scan repository ~/home/git/codebase --branch dev` +# MCP Command \[EXPERIMENT\] -## Monitor Option +> [!WARNING] +> The MCP command is available only for Python 3.10 and above. If you're using an earlier Python version, this command will not be available. -> :memo: **Note**
-> This option is only available to SCA scans. +The Model Context Protocol (MCP) command allows you to start an MCP server that exposes Cycode's scanning capabilities to AI systems and applications. This enables AI models to interact with Cycode CLI tools via a standardized protocol. -To push scan results tied to the [SCA policies](https://docs.cycode.com/docs/sca-policies) found in an SCA type scan to Cycode's knowledge graph, add the argument `--monitor` to the scan command. +> [!TIP] +> For the best experience, install Cycode CLI globally on your system using `pip install cycode` or `brew install cycode`, then authenticate once with `cycode auth`. After global installation and authentication, you won't need to configure `CYCODE_CLIENT_ID` and `CYCODE_CLIENT_SECRET` environment variables in your MCP configuration files. -Consider the following example. The following command will scan the repository for SCA policy violations and push them to Cycode: +[![Add MCP Server to Cursor using UV](https://cursor.com/deeplink/mcp-install-dark.svg)](https://cursor.com/en/install-mcp?name=cycode&config=eyJjb21tYW5kIjoidXZ4IGN5Y29kZSBtY3AiLCJlbnYiOnsiQ1lDT0RFX0NMSUVOVF9JRCI6InlvdXItY3ljb2RlLWlkIiwiQ1lDT0RFX0NMSUVOVF9TRUNSRVQiOiJ5b3VyLWN5Y29kZS1zZWNyZXQta2V5IiwiQ1lDT0RFX0FQSV9VUkwiOiJodHRwczovL2FwaS5jeWNvZGUuY29tIiwiQ1lDT0RFX0FQUF9VUkwiOiJodHRwczovL2FwcC5jeWNvZGUuY29tIn19) -`cycode scan -t sca --monitor repository ~/home/git/codebase` -or: +## Starting the MCP Server -`cycode scan --scan-type sca --monitor repository ~/home/git/codebase` +To start the MCP server, use the following command: -When using this option, the scan results from this scan will appear in the knowledge graph, which can be found [here](https://app.cycode.com/query-builder). +```bash +cycode mcp +``` -> :warning: **NOTE**
-> You must be an `owner` or an `admin` in Cycode to view the knowledge graph page. +By default, this starts the server using the `stdio` transport, which is suitable for local integrations and AI applications that can spawn subprocesses. -## Report Option +### Available Options -> :memo: **Note**
-> This option is only available to SCA scans. +| Option | Description | +|-------------------|--------------------------------------------------------------------------------------------| +| `-t, --transport` | Transport type for the MCP server: `stdio`, `sse`, or `streamable-http` (default: `stdio`) | +| `-H, --host` | Host address to bind the server (used only for non stdio transport) (default: `127.0.0.1`) | +| `-p, --port` | Port number to bind the server (used only for non stdio transport) (default: `8000`) | +| `--help` | Show help message and available options | -To push scan results tied to the [SCA policies](https://docs.cycode.com/docs/sca-policies) found in the Repository scan to Cycode, add the argument `--report` to the scan command. +### MCP Tools -`cycode scan -t sca --report repository ~/home/git/codebase` +The MCP server provides the following tools that AI systems can use: -or: +| Tool Name | Description | +|----------------------|---------------------------------------------------------------------------------------------| +| `cycode_secret_scan` | Scan files for hardcoded secrets | +| `cycode_sca_scan` | Scan files for Software Composition Analysis (SCA) - vulnerabilities and license issues | +| `cycode_iac_scan` | Scan files for Infrastructure as Code (IaC) misconfigurations | +| `cycode_sast_scan` | Scan files for Static Application Security Testing (SAST) - code quality and security flaws | +| `cycode_status` | Get Cycode CLI version, authentication status, and configuration information | -`cycode scan --scan-type sca --report repository ~/home/git/codebase` +### Usage Examples -When using this option, the scan results from this scan will appear in the On-Demand Scans section of Cycode. To get to this page, click the link that appears after the printed results: +#### Basic Command Examples -> :warning: **NOTE**
-> You must be an `owner` or an `admin` in Cycode to view this page. +Start the MCP server with default settings (stdio transport): +```bash +cycode mcp +``` +Start the MCP server with explicit stdio transport: ```bash -Scan Results: (scan_id: e04e06e5-6dd8-474f-b409-33bbee67270b) -⛔ Found issue of type: Security vulnerability in package 'vyper' referenced in project '': Multiple evaluation of contract address in call in vyper (rule ID: d003b23a-a2eb-42f3-83c9-7a84505603e5) in file: ./requirements.txt ⛔ +cycode mcp -t stdio +``` -1 | PyYAML~=5.3.1 -2 | vyper==0.3.1 -3 | cleo==1.0.0a5 +Start the MCP server with Server-Sent Events (SSE) transport: +```bash +cycode mcp -t sse -p 8080 +``` -⛔ Found issue of type: Security vulnerability in package 'vyper' referenced in project '': Integer bounds error in Vyper (rule ID: d003b23a-a2eb-42f3-83c9-7a84505603e5) in file: ./requirements.txt ⛔ +Start the MCP server with streamable HTTP transport on custom host and port: +```bash +cycode mcp -t streamable-http -H 0.0.0.0 -p 9000 +``` + +Learn more about MCP Transport types in the [MCP Protocol Specification – Transports](https://modelcontextprotocol.io/specification/2025-03-26/basic/transports). + +#### Configuration Examples + +##### Using MCP with Cursor/VS Code/Claude Desktop/etc (mcp.json) + +> [!NOTE] +> For EU Cycode environments, make sure to set the appropriate `CYCODE_API_URL` and `CYCODE_APP_URL` values in the environment variables (e.g., `https://api.eu.cycode.com` and `https://app.eu.cycode.com`). + +Follow [this guide](https://code.visualstudio.com/docs/copilot/chat/mcp-servers) to configure the MCP server in your **VS Code/GitHub Copilot**. Keep in mind that in `settings.json`, there is an `mcp` object containing a nested `servers` sub-object, rather than a standalone `mcpServers` object. + +For **stdio transport** (direct execution): +```json +{ + "mcpServers": { + "cycode": { + "command": "cycode", + "args": ["mcp"], + "env": { + "CYCODE_CLIENT_ID": "your-cycode-id", + "CYCODE_CLIENT_SECRET": "your-cycode-secret-key", + "CYCODE_API_URL": "https://api.cycode.com", + "CYCODE_APP_URL": "https://app.cycode.com" + } + } + } +} +``` + +For **stdio transport** with `pipx` installation: +```json +{ + "mcpServers": { + "cycode": { + "command": "pipx", + "args": ["run", "cycode", "mcp"], + "env": { + "CYCODE_CLIENT_ID": "your-cycode-id", + "CYCODE_CLIENT_SECRET": "your-cycode-secret-key", + "CYCODE_API_URL": "https://api.cycode.com", + "CYCODE_APP_URL": "https://app.cycode.com" + } + } + } +} +``` + +For **stdio transport** with `uvx` installation: +```json +{ + "mcpServers": { + "cycode": { + "command": "uvx", + "args": ["cycode", "mcp"], + "env": { + "CYCODE_CLIENT_ID": "your-cycode-id", + "CYCODE_CLIENT_SECRET": "your-cycode-secret-key", + "CYCODE_API_URL": "https://api.cycode.com", + "CYCODE_APP_URL": "https://app.cycode.com" + } + } + } +} +``` -1 | PyYAML~=5.3.1 -2 | vyper==0.3.1 -3 | cleo==1.0.0a5 +For **SSE transport** (Server-Sent Events): +```json +{ + "mcpServers": { + "cycode": { + "url": "http://127.0.0.1:8000/sse" + } + } +} +``` -⛔ Found issue of type: Security vulnerability in package 'pyyaml' referenced in project '': Improper Input Validation in PyYAML (rule ID: d003b23a-a2eb-42f3-83c9-7a84505603e5) in file: ./requirements.txt ⛔ +For **SSE transport** on custom port: +```json +{ + "mcpServers": { + "cycode": { + "url": "http://127.0.0.1:8080/sse" + } + } +} +``` -1 | PyYAML~=5.3.1 -2 | vyper==0.3.1 -3 | cleo==1.0.0a5 +For **streamable HTTP transport**: +```json +{ + "mcpServers": { + "cycode": { + "url": "http://127.0.0.1:8000/mcp" + } + } +} +``` -⛔ Found issue of type: Security vulnerability in package 'cleo' referenced in project '': cleo is vulnerable to Regular Expression Denial of Service (ReDoS) (rule ID: d003b23a-a2eb-42f3-83c9-7a84505603e5) in file: ./requirements.txt ⛔ +##### Running MCP Server in Background -2 | vyper==0.3.1 -3 | cleo==1.0.0a5 -4 | +For **SSE transport** (start server first, then configure client): +```bash +# Start the MCP server in the background +cycode mcp -t sse -p 8000 & + +# Configure in mcp.json +{ + "mcpServers": { + "cycode": { + "url": "http://127.0.0.1:8000/sse" + } + } +} +``` -⛔ Found issue of type: Security vulnerability in package 'vyper' referenced in project '': Incorrect Comparison in Vyper (rule ID: d003b23a-a2eb-42f3-83c9-7a84505603e5) in file: ./requirements.txt ⛔ +For **streamable HTTP transport**: +```bash +# Start the MCP server in the background +cycode mcp -t streamable-http -H 127.0.0.2 -p 9000 & + +# Configure in mcp.json +{ + "mcpServers": { + "cycode": { + "url": "http://127.0.0.2:9000/mcp" + } + } +} +``` -1 | PyYAML~=5.3.1 -2 | vyper==0.3.1 -3 | cleo==1.0.0a5 +> [!NOTE] +> The MCP server requires proper Cycode CLI authentication to function. Make sure you have authenticated using `cycode auth` or configured your credentials before starting the MCP server. -⛔ Found issue of type: Security vulnerability in package 'vyper' referenced in project '': Buffer Overflow in vyper (rule ID: d003b23a-a2eb-42f3-83c9-7a84505603e5) in file: ./requirements.txt ⛔ +### Troubleshooting MCP -1 | PyYAML~=5.3.1 -2 | vyper==0.3.1 -3 | cleo==1.0.0a5 +If you encounter issues with the MCP server, you can enable debug logging to get more detailed information about what's happening. There are two ways to enable debug logging: -Report URL: https://app.cycode.com/on-demand-scans/617ecc3d-9ff2-493e-8be8-2c1fecaf6939 +1. Using the `-v` or `--verbose` flag: +```bash +cycode -v mcp ``` +2. Using the `CYCODE_CLI_VERBOSE` environment variable: +```bash +CYCODE_CLI_VERBOSE=1 cycode mcp +``` + +The debug logs will show detailed information about: +- Server startup and configuration +- Connection attempts and status +- Tool execution and results +- Any errors or warnings that occur + +This information can be helpful when: +- Diagnosing connection issues +- Understanding why certain tools aren't working +- Identifying authentication problems +- Debugging transport-specific issues + + +# Scan Command + +## Running a Scan + +The Cycode CLI application offers several types of scans so that you can choose the option that best fits your case. The following are the current options and commands available: + +| Option | Description | +|------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------| +| `-t, --scan-type [secret\|iac\|sca\|sast]` | Specify the scan you wish to execute (`secret`/`iac`/`sca`/`sast`), the default is `secret`. | +| `--show-secret BOOLEAN` | Show secrets in plain text. See [Show/Hide Secrets](#showhide-secrets) section for more details. | +| `--soft-fail BOOLEAN` | Run scan without failing, always return a non-error status code. See [Soft Fail](#soft-fail) section for more details. | +| `--severity-threshold [INFO\|LOW\|MEDIUM\|HIGH\|CRITICAL]` | Show only violations at the specified level or higher. | +| `--sca-scan` | Specify the SCA scan you wish to execute (`package-vulnerabilities`/`license-compliance`). The default is both. | +| `--monitor` | When specified, the scan results will be recorded in Cycode. | +| `--cycode-report` | Display a link to the scan report in the Cycode platform in the console output. | +| `--no-restore` | When specified, Cycode will not run the restore command. This will scan direct dependencies ONLY! | +| `--gradle-all-sub-projects` | Run gradle restore command for all sub projects. This should be run from | +| `--maven-settings-file` | For Maven only, allows using a custom [settings.xml](https://maven.apache.org/settings.html) file when scanning for dependencies | +| `--help` | Show options for given command. | + +| Command | Description | +|----------------------------------------|-----------------------------------------------------------------------| +| [commit-history](#commit-history-scan) | Scan commit history or perform diff scanning between specific commits | +| [path](#path-scan) | Scan the files in the path supplied in the command | +| [pre-commit](#pre-commit-scan) | Use this command to scan the content that was not committed yet | +| [repository](#repository-scan) | Scan git repository including its history | + +### Options + +#### Severity Option + +To limit the results of the scan to a specific severity threshold, the argument `--severity-threshold` can be added to the scan command. + +For example, the following command will scan the repository for policy violations that have severity of Medium or higher: + +`cycode scan --severity-threshold MEDIUM repository ~/home/git/codebase` + +#### Monitor Option + +> [!NOTE] +> This option is only available to SCA scans. + +To push scan results tied to the [SCA policies](https://docs.cycode.com/docs/sca-policies) found in an SCA type scan to Cycode, add the argument `--monitor` to the scan command. + +For example, the following command will scan the repository for SCA policy violations and push them to Cycode platform: + +`cycode scan -t sca --monitor repository ~/home/git/codebase` + +#### Cycode Report Option + +For every scan performed using the Cycode CLI, a report is automatically generated and its results are sent to Cycode. These results are tied to the relevant policies (e.g., [SCA policies](https://docs.cycode.com/docs/sca-policies) for Repository scans) within the Cycode platform. + +To have the direct URL to this Cycode report printed in your CLI output after the scan completes, add the argument `--cycode-report` to your scan command. + +`cycode scan --cycode-report repository ~/home/git/codebase` + +All scan results from the CLI will appear in the CLI Logs section of Cycode. If you included the `--cycode-report` flag in your command, a direct link to the specific report will be displayed in your terminal following the scan results. + +> [!WARNING] +> You must have the `owner` or `admin` role in Cycode to view this page. + +![cli-report](https://raw.githubusercontent.com/cycodehq/cycode-cli/main/images/sca_report_url.png) + The report page will look something like below: -![](https://raw.githubusercontent.com/cycodehq-public/cycode-cli/main/images/scan_details.png) +![](https://raw.githubusercontent.com/cycodehq/cycode-cli/main/images/scan_details.png) -## Package Vulnerabilities Option +#### Package Vulnerabilities Option -> :memo: **Note**
+> [!NOTE] > This option is only available to SCA scans. To scan a specific package vulnerability of your local repository, add the argument `--sca-scan package-vulnerabilities` following the `-t sca` or `--scan-type sca` option. -Consider the previous example. If you wanted to only run an SCA scan on package vulnerabilities, you could execute the following: +In the previous example, if you wanted to only run an SCA scan on package vulnerabilities, you could execute the following: `cycode scan -t sca --sca-scan package-vulnerabilities repository ~/home/git/codebase` -or: - -`cycode scan --scan-type sca --sca-scan package-vulnerabilities repository ~/home/git/codebase` +#### License Compliance Option -### License Compliance Option - -> :memo: **Note**
+> [!NOTE] > This option is only available to SCA scans. To scan a specific branch of your local repository, add the argument `--sca-scan license-compliance` followed by the name of the branch you wish to scan. -Consider the previous example. If you wanted to only scan a branch named `dev`, you could execute the following: +In the previous example, if you wanted to only scan a branch named `dev`, you could execute the following: `cycode scan -t sca --sca-scan license-compliance repository ~/home/git/codebase -b dev` -or: +#### Lock Restore Option -`cycode scan --scan-type sca --sca-scan license-compliance repository ~/home/git/codebase` +> [!NOTE] +> This option is only available to SCA scans. -### Severity Threshold +When running an SCA scan, Cycode CLI automatically attempts to restore (generate) a dependency lockfile for each supported manifest file it finds. This allows scanning transitive dependencies, not just the ones listed directly in the manifest. To skip this step and scan only direct dependencies, use the `--no-restore` flag. -> :memo: **Note**
-> This option is only available to SCA scans. +The following ecosystems support automatic lockfile restoration: + +| Ecosystem | Manifest file | Lockfile generated | Tool invoked (when lockfile is absent) | +|---|---|---|---| +| npm | `package.json` | `package-lock.json` | `npm install --package-lock-only --ignore-scripts --no-audit` | +| Yarn | `package.json` | `yarn.lock` | `yarn install --ignore-scripts` | +| pnpm | `package.json` | `pnpm-lock.yaml` | `pnpm install --ignore-scripts` | +| Deno | `deno.json` / `deno.jsonc` | `deno.lock` | *(read existing lockfile only)* | +| Go | `go.mod` | `go.mod.graph` | `go list -m -json all` + `go mod graph` | +| Maven | `pom.xml` | `bcde.mvndeps` | `mvn dependency:tree` | +| Gradle | `build.gradle` / `build.gradle.kts` | `gradle-dependencies-generated.txt` | `gradle dependencies -q --console plain` | +| SBT | `build.sbt` | `build.sbt.lock` | `sbt dependencyLockWrite` | +| NuGet | `*.csproj` | `packages.lock.json` | `dotnet restore --use-lock-file` | +| Ruby | `Gemfile` | `Gemfile.lock` | `bundle --quiet` | +| Poetry | `pyproject.toml` | `poetry.lock` | `poetry lock` | +| Pipenv | `Pipfile` | `Pipfile.lock` | `pipenv lock` | +| PHP Composer | `composer.json` | `composer.lock` | `composer update --no-cache --no-install --no-scripts --ignore-platform-reqs` | + +If a lockfile already exists alongside the manifest, Cycode reads it directly without running any install command. + +**SBT prerequisite:** The `sbt-dependency-lock` plugin must be installed. Add the following line to `project/plugins.sbt`: + +```text +addSbtPlugin("software.purpledragon" % "sbt-dependency-lock" % "1.5.1") +``` + +### Repository Scan + +A repository scan examines an entire local repository for any exposed secrets or insecure misconfigurations. This more holistic scan type looks at everything: the current state of your repository and its commit history. It will look not only for secrets that are currently exposed within the repository but previously deleted secrets as well. + +To execute a full repository scan, execute the following: + +`cycode scan repository {{path}}` + +For example, if you wanted to scan a repository stored in `~/home/git/codebase`, you could execute the following: + +`cycode scan repository ~/home/git/codebase` + +The following option is available for use with this command: -To limit the results of the `sca` scan to a specific severity threshold, add the argument `--severity-threshold` to the scan command. +| Option | Description | +|---------------------|--------------------------------------------------------| +| `-b, --branch TEXT` | Branch to scan, if not set scanning the default branch | -Consider the following example. The following command will scan the repository for SCA policy violations that have a severity of Medium or higher: +#### Branch Option -`cycode scan -t sca --security-threshold MEDIUM repository ~/home/git/codebase` +To scan a specific branch of your local repository, add the argument `-b` (alternatively, `--branch`) followed by the name of the branch you wish to scan. -or: +Given the previous example, if you wanted to only scan a branch named `dev`, you could execute the following: -`cycode scan --scan-type sca --security-threshold MEDIUM repository ~/home/git/codebase` +`cycode scan repository ~/home/git/codebase -b dev` -## Path Scan +### Path Scan A path scan examines a specific local directory and all the contents within it, instead of focusing solely on a GIT repository. @@ -419,165 +734,385 @@ For example, consider a scenario in which you want to scan the directory located `cycode scan path ~/home/git/codebase` -## Commit History Scan +#### Terraform Plan Scan + +Cycode CLI supports Terraform plan scanning (supporting Terraform 0.12 and later) + +Terraform plan file must be in JSON format (having `.json` extension) + +If you just have a configuration file, you can generate a plan by doing the following: + +1. Initialize a working directory that contains Terraform configuration file: + + `terraform init` + +2. Create Terraform execution plan and save the binary output: + + `terraform plan -out={tfplan_output}` + +3. Convert the binary output file into readable JSON: + + `terraform show -json {tfplan_output} > {tfplan}.json` + +4. Scan your `{tfplan}.json` with Cycode CLI: + + `cycode scan -t iac path ~/PATH/TO/YOUR/{tfplan}.json` + +### Commit History Scan -A commit history scan is limited to a local repository’s previous commits, focused on finding any secrets within the commit history, instead of examining the repository’s current state. +> [!NOTE] +> Commit History Scan is not available for IaC scans. + +The commit history scan command provides two main capabilities: + +1. **Full History Scanning**: Analyze all commits in the repository history +2. **Diff Scanning**: Scan only the changes between specific commits + +Secrets scanning can analyze all commits in the repository history because secrets introduced and later removed can still be leaked or exposed. For SCA and SAST scans, the commit history command focuses on scanning the differences/changes between commits, making it perfect for pull request reviews and incremental scanning. + +A commit history scan examines your Git repository's commit history and can be used both for comprehensive historical analysis and targeted diff scanning of specific changes. To execute a commit history scan, execute the following: -`cycode scan commit_history {{path}}` +`cycode scan commit-history {{path}}` For example, consider a scenario in which you want to scan the commit history for a repository stored in `~/home/git/codebase`. You could then execute the following: -`cycode scan commit_history ~/home/git/codebase` +`cycode scan commit-history ~/home/git/codebase` The following options are available for use with this command: -| Option | Description | -|---------------------------|-------------| -| `-r, --commit_range TEXT` | Scan a commit range in this git repository, by default cycode scans all commit history (example: HEAD~1) | +| Option | Description | +|---------------------------|----------------------------------------------------------------------------------------------------------| +| `-r, --commit-range TEXT` | Scan a commit range in this git repository, by default cycode scans all commit history (example: HEAD~1) | -### Commit Range Option +#### Commit Range Option (Diff Scanning) -The commit history scan, by default, examines the repository’s entire commit history, all the way back to the initial commit. You can instead limit the scan to a specific commit range by adding the argument `--commit_range` followed by the name you specify. +The commit range option enables **diff scanning** – scanning only the changes between specific commits instead of the entire repository history. +This is particularly useful for: +- **Pull request validation**: Scan only the changes introduced in a PR +- **Incremental CI/CD scanning**: Focus on recent changes rather than the entire codebase +- **Feature branch review**: Compare changes against main/master branch +- **Performance optimization**: Faster scans by limiting scope to relevant changes -Consider the previous example. If you wanted to scan only specific commits on your repository, you could execute the following: +#### Commit Range Syntax -`cycode scan commit_history -r {{from-commit-id}}...{{to-commit-id}} ~/home/git/codebase` +The `--commit-range` (`-r`) option supports standard Git revision syntax: -OR +| Syntax | Description | Example | +|---------------------|-----------------------------------|-------------------------| +| `commit1..commit2` | Changes from commit1 to commit2 | `abc123..def456` | +| `commit1...commit2` | Changes in commit2 not in commit1 | `main...feature-branch` | +| `commit` | Changes from commit to HEAD | `HEAD~1` | +| `branch1..branch2` | Changes from branch1 to branch2 | `main..feature-branch` | -`cycode scan commit_history --commit_range {{from-commit-id}}...{{to-commit-id}} ~/home/git/codebase` +#### Diff Scanning Examples -## Pre-Commit Scan +**Scan changes in the last commit:** +```bash +cycode scan commit-history -r HEAD~1 ~/home/git/codebase +``` -A pre-commit scan automatically identifies any issues before you commit changes to your repository. There is no need to manually execute this scan; simply configure the pre-commit hook as detailed under the Installation section of this guide. +**Scan changes between two specific commits:** +```bash +cycode scan commit-history -r abc123..def456 ~/home/git/codebase +``` -After your install the pre-commit hook and, you may, on occasion, wish to skip scanning during a specific commit. Simply add the following to your `git` command to skip scanning for a single commit: +**Scan changes in your feature branch compared to main:** +```bash +cycode scan commit-history -r main..HEAD ~/home/git/codebase +``` -`SKIP=cycode git commit -m ` +**Scan changes between main and a feature branch:** +```bash +cycode scan commit-history -r main..feature-branch ~/home/git/codebase +``` -# Scan Results +**Scan all changes in the last 3 commits:** +```bash +cycode scan commit-history -r HEAD~3..HEAD ~/home/git/codebase +``` -Each scan will complete with a message stating if any issues were found or not. +> [!TIP] +> For CI/CD pipelines, you can use environment variables like `${{ github.event.pull_request.base.sha }}..${{ github.sha }}` (GitHub Actions) or `$CI_MERGE_REQUEST_TARGET_BRANCH_SHA..$CI_COMMIT_SHA` (GitLab CI) to scan only PR/MR changes. -If no issues are found, the scan ends with the following success message: +### Pre-Commit Scan -`Good job! No issues were found!!! 👏👏👏` +A pre-commit scan automatically identifies any issues before you commit changes to your repository. There is no need to manually execute this scan; configure the pre-commit hook as detailed under the Installation section of this guide. -If an issue is found, a `Found issue of type:` message appears upon completion instead: +After installing the pre-commit hook, you may occasionally wish to skip scanning during a specific commit. To do this, add the following to your `git` command to skip scanning for a single commit: ```bash -⛔ Found issue of type: generic-password (rule ID: ce3a4de0-9dfc-448b-a004-c538cf8b4710) in file: config/my_config.py -Secret SHA: a44081db3296c84b82d12a35c446a3cba19411dddfa0380134c75f7b3973bff0 ⛔ -0 | @@ -0,0 +1 @@ -1 | +my_password = 'h3l***********350' -2 | \ No newline at end of file +SKIP=cycode git commit -m ` ``` -In the event an issue is found, review the file in question for the specific line highlighted by the result message. Implement any changes required to resolve the issue, then execute the scan again. +### Pre-Push Scan -## Show/Hide Secrets +A pre-push scan automatically identifies any issues before you push changes to the remote repository. This hook runs on the client side and scans only the commits that are about to be pushed, making it efficient for catching issues before they reach the remote repository. -In the above example, a secret was found in the file `secret_test`, located in the subfolder `cli`. The second part of the message shows the specific line the secret appears in, which in this case is a value assigned to `googleApiKey`. +> [!NOTE] +> Pre-push hook is not available for IaC scans. -Note how the above example obscures the actual secret value, replacing most of the secret with asterisks. Scans obscure secrets by default, but you may optionally disable this feature in order to view the full secret (assuming the machine you are viewing the scan result on is sufficiently secure from prying eyes). +The pre-push hook integrates with the pre-commit framework and can be configured to run before any `git push` operation. -To disable secret obfuscation, add the `--show-secret` argument to any type of scan, then assign it a `1` value to show the full secret in the result message, or `0` to hide the secret (which is done by default). +#### Installing Pre-Push Hook -In the following example, a Path Scan is executed against the `cli` subdirectory with the option enabled to display any secrets found in full: +To set up the pre-push hook using the pre-commit framework: -`cycode scan --show-secret=1 path ./cli` +1. Install the pre-commit framework (if not already installed): -The result would then not be obfuscated: + ```bash + pip3 install pre-commit + ``` -```bash -⛔ Found issue of type: generic-password (rule ID: ce3a4de0-9dfc-448b-a004-c538cf8b4710) in file: config/my_config.py -Secret SHA: a44081db3296c84b82d12a35c446a3cba19411dddfa0380134c75f7b3973bff0 ⛔ -0 | @@ -0,0 +1 @@ -1 | +my_password = 'h3110w0r1d!@#$350' -2 | \ No newline at end of file -``` +2. Create or update your `.pre-commit-config.yaml` file to include the pre-push hooks: + + ```yaml + repos: + - repo: https://github.com/cycodehq/cycode-cli + rev: v3.5.0 + hooks: + - id: cycode-pre-push + stages: [pre-push] + ``` + +3. For multiple scan types, use this configuration: + + ```yaml + repos: + - repo: https://github.com/cycodehq/cycode-cli + rev: v3.5.0 + hooks: + - id: cycode-pre-push # Secrets scan + stages: [pre-push] + - id: cycode-sca-pre-push # SCA scan + stages: [pre-push] + - id: cycode-sast-pre-push # SAST scan + stages: [pre-push] + ``` + +4. Install the pre-push hook: -## Soft Fail + ```bash + pre-commit install --hook-type pre-push + ``` -Utilizing the soft fail feature will not fail the CI/CD step within the pipeline if the Cycode scan finds an issue. Additionally, in case an issue occurs from Cycode’s side, a soft fail will automatically execute to avoid interference. + A successful installation will result in the message: `Pre-push installed at .git/hooks/pre-push`. -Add the `--soft-fail` argument to any type of scan to configure this feature, then assign a value of `1` if you want found issues to result in a failure within the CI/CD tool or `0` for scan results to have no impact (result in a `success` result). +5. Keep the pre-push hook up to date: -## Example Scan Results + ```bash + pre-commit autoupdate + ``` -### Secrets Result Example +#### How Pre-Push Scanning Works +The pre-push hook: +- Receives information about what commits are being pushed +- Calculates the appropriate commit range to scan +- For new branches: scans all commits from the merge base with the default branch +- For existing branches: scans only the new commits since the last push +- Runs the same comprehensive scanning as other Cycode scan modes + +#### Smart Default Branch Detection + +The pre-push hook intelligently detects the default branch for merge base calculation using this priority order: + +1. **Environment Variable**: `CYCODE_DEFAULT_BRANCH` - allows manual override +2. **Git Remote HEAD**: Uses `git symbolic-ref refs/remotes/origin/HEAD` to detect the actual remote default branch +3. **Git Remote Info**: Falls back to `git remote show origin` if symbolic-ref fails +4. **Hardcoded Fallbacks**: Uses common default branch names (origin/main, origin/master, main, master) + +**Setting a Custom Default Branch:** ```bash -⛔ Found issue of type: generic-password (rule ID: ce3a4de0-9dfc-448b-a004-c538cf8b4710) in file: config/my_config.py -Secret SHA: a44081db3296c84b82d12a35c446a3cba19411dddfa0380134c75f7b3973bff0 ⛔ -0 | @@ -0,0 +1 @@ -1 | +my_password = 'h3l***********350' -2 | \ No newline at end of file +export CYCODE_DEFAULT_BRANCH=origin/develop ``` -### IaC Result Example +This smart detection ensures the pre-push hook works correctly regardless of whether your repository uses `main`, `master`, `develop`, or any other default branch name. + +#### Skipping Pre-Push Scans + +To skip the pre-push scan for a specific push operation, use: ```bash -⛔ Found issue of type: Resource should use non-default namespace (rule ID: bdaa88e2-5e7c-46ff-ac2a-29721418c59c) in file: ./k8s/k8s.yaml ⛔ +SKIP=cycode-pre-push git push +``` + +Or to skip all pre-push hooks: -7 | name: secrets-file -8 | namespace: default -9 | resourceVersion: "4228" +```bash +git push --no-verify ``` -### SCA Result Example +> [!TIP] +> The pre-push hook is triggered on `git push` command and scans only the commits that are about to be pushed, making it more efficient than scanning the entire repository. + +## Exclude Paths From Scans +You can use a `.cycodeignore` file to tell the Cycode CLI which files and directories to exclude from scans. +It works just like a `.gitignore` file. This helps you focus scans on your relevant code and prevent certain paths from triggering violations locally. + +### How It Works +1. Create a file named `.cycodeignore` in your workfolder. +2. List the files and directories you want to exclude, using the same patterns as `.gitignore`. +3. Place this file in the directory where you plan to run the cycode scan command. + +> [!WARNING] +> - **Invalid files**: If the `.cycodeignore` file contains a syntax error, the CLI scan will fail and return an error. +> - **Ignoring paths vs. violations**: This file is for excluding paths. It's different from the CLI's capability to ignore specific violations (for example, by using the --ignore-violation flag). + +### Supported Scanners +- SAST +- IaC (comming soon) +- SCA (comming soon) + +## Scan Results + +Each scan will complete with a message stating if any issues were found or not. + +If no issues are found, the scan ends with the following success message: + +`Good job! No issues were found!!! 👏👏👏` + +If an issue is found, a violation card appears upon completion instead. In this case you should review the file in question for the specific line highlighted by the result message. Implement any changes required to resolve the issue, then execute the scan again. + +### Show/Hide Secrets + +In the [examples below](#secrets-result-example), a secret was found in the file `secret_test`, located in the subfolder `cli`. The second part of the message shows the specific line the secret appears in, which in this case is a value assigned to `googleApiKey`. + +Note how the example obscures the actual secret value, replacing most of the secret with asterisks. Scans obscure secrets by default, but you may optionally disable this feature to view the full secret (assuming the machine you are viewing the scan result on is sufficiently secure from prying eyes). + +To disable secret obfuscation, add the `--show-secret` argument to any type of scan. + +In the following example, a Path Scan is executed against the `cli` subdirectory with the option enabled to display any secrets found in full: + +`cycode scan --show-secret path ./cli` + +The result would then not be obfuscated. + +### Soft Fail + +In normal operation the CLI will return an exit code of `1` when issues are found in the scan results. Depending on your CI/CD setup this will usually result in an overall failure. If you don't want this to happen, you can use the soft fail feature. + +By adding the `--soft-fail` option to any type of scan, the exit code will be forced to `0` regardless of whether any results are found. + +### Example Scan Results + +#### Secrets Result Example ```bash -⛔ Found issue of type: Security vulnerability in package 'pyyaml' referenced in project 'Users/myuser/my-test-repo': Improper Input Validation in PyYAML (rule ID: d003b23a-a2eb-42f3-83c9-7a84505603e5) in file: Users/myuser/my-test-repo/requirements.txt ⛔ +╭─────────────────────────────────────────────────────────────── Hardcoded generic-password is used ───────────────────────────────────────────────────────────────╮ +│ Violation 12 of 12 │ +│ ╭─ 🔍 Details ───────────────────────────────────────╮ ╭─ 💻 Code Snippet ─────────────────────────────────────────────────────────────────────────────────────╮ │ +│ │ Severity 🟠 MEDIUM │ │ 34 }; │ │ +│ │ In file /Users/cycodemacuser/NodeGoat/test/s │ │ 35 │ │ +│ │ ecurity/profile-test.js │ │ 36 var sutUserName = "user1"; │ │ +│ │ Secret SHA b4ea3116d868b7c982ee6812cce61727856b │ │ ❱ 37 var sutUserPassword = "Us*****23"; │ │ +│ │ 802b3063cd5aebe7d796988552e0 │ │ 38 │ │ +│ │ Rule ID 68b6a876-4890-4e62-9531-0e687223579f │ │ 39 chrome.setDefaultService(service); │ │ +│ ╰────────────────────────────────────────────────────╯ │ 40 │ │ +│ ╰───────────────────────────────────────────────────────────────────────────────────────────────────────╯ │ +│ ╭─ 📝 Summary ─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ │ +│ │ A generic secret or password is an authentication token used to access a computer or application and is assigned to a password variable. │ │ +│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +``` -1 | PyYAML~=5.3.1 -2 | vyper==0.3.1 -3 | cleo==1.0.0a5 +#### IaC Result Example + +```bash +╭──────────── Enable Content Encoding through the attribute 'MinimumCompressionSize'. This value should be greater than -1 and smaller than 10485760. ─────────────╮ +│ Violation 45 of 110 │ +│ ╭─ 🔍 Details ───────────────────────────────────────╮ ╭─ 💻 Code Snippet ─────────────────────────────────────────────────────────────────────────────────────╮ │ +│ │ Severity 🟠 MEDIUM │ │ 20 BinaryMediaTypes: │ │ +│ │ In file ...ads-copy/iac/cft/api-gateway/ap │ │ 21 - !Ref binaryMediaType1 │ │ +│ │ i-gateway-rest-api/deploy.yml │ │ 22 - !Ref binaryMediaType2 │ │ +│ │ IaC Provider CloudFormation │ │ ❱ 23 MinimumCompressionSize: -1 │ │ +│ │ Rule ID 33c4b90c-3270-4337-a075-d3109c141b │ │ 24 EndpointConfiguration: │ │ +│ │ 53 │ │ 25 Types: │ │ +│ ╰────────────────────────────────────────────────────╯ │ 26 - EDGE │ │ +│ ╰───────────────────────────────────────────────────────────────────────────────────────────────────────╯ │ +│ ╭─ 📝 Summary ─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ │ +│ │ This policy validates the proper configuration of content encoding in AWS API Gateway. Specifically, the policy checks for the attribute │ │ +│ │ 'minimum_compression_size' in API Gateway REST APIs. Correct configuration of this attribute is important for enabling content encoding of API responses for │ │ +│ │ improved API performance and reduced payload sizes. │ │ +│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ``` -### SAST Result Example +#### SCA Result Example ```bash -⛔ Found issue of type: Detected a request using 'http://'. This request will be unencrypted, and attackers could listen into traffic on the network and be able to obtain sensitive information. Use 'https://' instead. (rule ID: 3fbbd34b-b00d-4415-b9d9-f861c076b9f2) in file: ./requests.py ⛔ +╭─────────────────────────────────────────────────────── [CVE-2019-10795] Prototype Pollution in undefsafe ────────────────────────────────────────────────────────╮ +│ Violation 172 of 195 │ +│ ╭─ 🔍 Details ───────────────────────────────────────╮ ╭─ 💻 Code Snippet ─────────────────────────────────────────────────────────────────────────────────────╮ │ +│ │ Severity 🟠 MEDIUM │ │ 26758 "integrity": "sha1-5z3T17DXxe2G+6xrCufYxqadUPo=", │ │ +│ │ In file /Users/cycodemacuser/Node │ │ 26759 "dev": true │ │ +│ │ Goat/package-lock.json │ │ 26760 }, │ │ +│ │ CVEs CVE-2019-10795 │ │ ❱ 26761 "undefsafe": { │ │ +│ │ Package undefsafe │ │ 26762 "version": "2.0.2", │ │ +│ │ Version 2.0.2 │ │ 26763 "resolved": "https://registry.npmjs.org/undefsafe/-/undefsafe-2.0.2.tgz", │ │ +│ │ First patched version Not fixed │ │ 26764 "integrity": "sha1-Il9rngM3Zj4Njnz9aG/Cg2zKznY=", │ │ +│ │ Dependency path nodemon 1.19.1 -> │ ╰───────────────────────────────────────────────────────────────────────────────────────────────────────╯ │ +│ │ undefsafe 2.0.2 │ │ +│ │ Rule ID 9c6a8911-e071-4616-86db-4 │ │ +│ │ 943f2e1df81 │ │ +│ ╰────────────────────────────────────────────────────╯ │ +│ ╭─ 📝 Summary ─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ │ +│ │ undefsafe before 2.0.3 is vulnerable to Prototype Pollution. The 'a' function could be tricked into adding or modifying properties of Object.prototype using │ │ +│ │ a __proto__ payload. │ │ +│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +``` + +#### SAST Result Example -2 | -3 | res = requests.get('http://example.com', timeout=1) -4 | print(res.content) +```bash +╭───────────────────────────────────────────── [CWE-208: Observable Timing Discrepancy] Observable Timing Discrepancy ─────────────────────────────────────────────╮ +│ Violation 24 of 49 │ +│ ╭─ 🔍 Details ───────────────────────────────────────╮ ╭─ 💻 Code Snippet ─────────────────────────────────────────────────────────────────────────────────────╮ │ +│ │ Severity 🟠 MEDIUM │ │ 173 " including numbers, lowercase and uppercase letters."; │ │ +│ │ In file /Users/cycodemacuser/NodeGoat/app │ │ 174 return false; │ │ +│ │ /routes/session.js │ │ 175 } │ │ +│ │ CWE CWE-208 │ │ ❱ 176 if (password !== verify) { │ │ +│ │ Subcategory Security │ │ 177 errors.verifyError = "Password must match"; │ │ +│ │ Language js │ │ 178 return false; │ │ +│ │ Security Tool Bearer (Powered by Cycode) │ │ 179 } │ │ +│ │ Rule ID 19fbca07-a8e7-4fa6-92ac-a36d15509 │ ╰───────────────────────────────────────────────────────────────────────────────────────────────────────╯ │ +│ │ fa9 │ │ +│ ╰────────────────────────────────────────────────────╯ │ +│ ╭─ 📝 Summary ─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ │ +│ │ Observable Timing Discrepancy occurs when the time it takes for certain operations to complete can be measured and observed by attackers. This vulnerability │ │ +│ │ is particularly concerning when operations involve sensitive information, such as password checks or secret comparisons. If attackers can analyze how long │ │ +│ │ these operations take, they might be able to deduce confidential details, putting your data at risk. │ │ +│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ``` -# Ignoring Scan Results +### Company Custom Remediation Guidelines -Ignore rules can be added to ignore specific secret values, specific SHA512 values, specific paths, and specific Cycode secret and IaC rule IDs. This will cause the scan to not alert these values. The ignore rules are written and saved locally in the `./.cycode/config.yaml` file. +If your company has set custom remediation guidelines in the relevant policy via the Cycode portal, you'll see a field for “Company Guidelines” that contains the remediation guidelines you added. Note that if you haven't added any company guidelines, this field will not appear in the CLI tool. -> :warning: **Warning**
-> Adding values to be ignored should be done with careful consideration of the values, paths, and policies to ensure that the scans will pick up true positives. -The following are the options available for the `cycode ignore` command: +## Ignoring Scan Results -| Option | Description | -|---------------------------------|-------------| -| `--by-value TEXT` | Ignore a specific value while scanning for secrets. See [Ignoring a Secret Value](#ignoring-a-secret-value) for more details. | -| `--by-sha TEXT` | Ignore a specific SHA512 representation of a string while scanning for secrets. See [Ignoring a Secret SHA Value](#ignoring-a-secret-sha-value) for more details. | -| `--by-path TEXT` | Avoid scanning a specific path. Need to specify scan type. See [Ignoring a Path](#ignoring-a-path) for more details. | -| `--by-rule TEXT` | Ignore scanning a specific secret rule ID/IaC rule ID/SCA rule ID. See [Ignoring a Secret or Iac Rule](#ignoring-a-secret-or-iac-rule) for more details. | -| `--by-package TEXT` | Ignore scanning a specific package version while running an SCA scan. Expected pattern - `name@version`. See [Ignoring a Package](#ignoring-a-package) for more details. | -| `-t, --scan-type [secret\|iac\|sca\|sast]` | Specify the scan you wish to execute (`secret`/`iac`/`sca`/`sast`), The default value is `secret` | -| `-g, --global` | Add an ignore rule and update it in the global `.cycode` config file | +Ignore rules can be added to ignore specific secret values, specific SHA512 values, specific paths, and specific Cycode secret and IaC rule IDs. This will cause the scan to not alert these values. The ignoring rules are written and saved locally in the `./.cycode/config.yaml` file. -In the following example, a pre-commit scan runs and finds the following: +> [!WARNING] +> Adding values to be ignored should be done with careful consideration of the values, paths, and policies to ensure that the scans will pick up true positives. -```bash -⛔ Found issue of type: generic-password (rule ID: ce3a4de0-9dfc-448b-a004-c538cf8b4710) in file: config/my_config.py -Secret SHA: a44081db3296c84b82d12a35c446a3cba19411dddfa0380134c75f7b3973bff0 ⛔ -0 | @@ -0,0 +1 @@ -1 | +my_password = 'h3l***********350' -2 | \ No newline at end of file -``` +The following are the options available for the `cycode ignore` command: -If this is a value that is not a valid secret, then use the the `cycode ignore` command to ignore the secret by its value, SHA value, specific path, or rule ID. If this is an IaC scan, then you can ignore that result by its path or rule ID. +| Option | Description | +|--------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `--by-value TEXT` | Ignore a specific value while scanning for secrets. See [Ignoring a Secret Value](#ignoring-a-secret-value) for more details. | +| `--by-sha TEXT` | Ignore a specific SHA512 representation of a string while scanning for secrets. See [Ignoring a Secret SHA Value](#ignoring-a-secret-sha-value) for more details. | +| `--by-path TEXT` | Avoid scanning a specific path. Need to specify scan type. See [Ignoring a Path](#ignoring-a-path) for more details. | +| `--by-rule TEXT` | Ignore scanning a specific secret rule ID/IaC rule ID/SCA rule ID. See [Ignoring a Secret or Iac Rule](#ignoring-a-secret-iac-sca-or-sast-rule) for more details. | +| `--by-package TEXT` | Ignore scanning a specific package version while running an SCA scan. Expected pattern - `name@version`. See [Ignoring a Package](#ignoring-a-package) for more details. | +| `--by-cve TEXT` | Ignore scanning a specific CVE while running an SCA scan. Expected pattern: CVE-YYYY-NNN. | +| `-t, --scan-type [secret\|iac\|sca\|sast]` | Specify the scan you wish to execute (`secret`/`iac`/`sca`/`sast`). The default value is `secret`. | +| `-g, --global` | Add an ignore rule and update it in the global `.cycode` config file. | -## Ignoring a Secret Value +### Ignoring a Secret Value To ignore a specific secret value, you will need to use the `--by-value` flag. This will ignore the given secret value from all future scans. Use the following command to add a secret value to be ignored: @@ -589,7 +1124,7 @@ In the example at the top of this section, the command to ignore a specific secr In the example above, replace the `h3110w0r1d!@#$350` value with your non-masked secret value. See the Cycode scan options for details on how to see secret values in the scan results. -## Ignoring a Secret SHA Value +### Ignoring a Secret SHA Value To ignore a specific secret SHA value, you will need to use the `--by-sha` flag. This will ignore the given secret SHA value from all future scans. Use the following command to add a secret SHA value to be ignored: @@ -601,16 +1136,12 @@ In the example at the top of this section, the command to ignore a specific secr In the example above, replace the `a44081db3296c84b82d12a35c446a3cba19411dddfa0380134c75f7b3973bff0` value with your secret SHA value. -## Ignoring a Path +### Ignoring a Path To ignore a specific path for either secret, IaC, or SCA scans, you will need to use the `--by-path` flag in conjunction with the `-t, --scan-type` flag (you must specify the scan type). This will ignore the given path from all future scans for the given scan type. Use the following command to add a path to be ignored: `cycode ignore -t {{scan-type}} --by-path {{path}}` -OR - -`cycode ignore --scan-type {{scan-type}} --by-path {{path}}` - In the example at the top of this section, the command to ignore a specific path for a secret is as follows: `cycode ignore -t secret --by-path ~/home/my-repo/config` @@ -629,37 +1160,33 @@ In the example at the top of this section, the command to ignore a specific path In the example above, replace the `~/home/my-repo/config` value with your path value. -## Ignoring a Secret, IaC, SCA, or SAST Rule +### Ignoring a Secret, IaC, SCA, or SAST Rule To ignore a specific secret, IaC, SCA, or SAST rule, you will need to use the `--by-rule` flag in conjunction with the `-t, --scan-type` flag (you must specify the scan type). This will ignore the given rule ID value from all future scans. Use the following command to add a rule ID value to be ignored: `cycode ignore -t {{scan-type}} --by-rule {{rule-ID}}` -OR - -`cycode ignore --scan-type {{scan-type}} --by-rule {{rule-ID}}` - In the example at the top of this section, the command to ignore the specific secret rule ID is as follows: -`cycode ignore --scan-type secret --by-rule ce3a4de0-9dfc-448b-a004-c538cf8b4710` +`cycode ignore -t secret --by-rule ce3a4de0-9dfc-448b-a004-c538cf8b4710` In the example above, replace the `ce3a4de0-9dfc-448b-a004-c538cf8b4710` value with the rule ID you want to ignore. In the example at the top of this section, the command to ignore the specific IaC rule ID is as follows: -`cycode ignore --scan-type iac --by-rule bdaa88e2-5e7c-46ff-ac2a-29721418c59c` +`cycode ignore -t iac --by-rule bdaa88e2-5e7c-46ff-ac2a-29721418c59c` In the example above, replace the `bdaa88e2-5e7c-46ff-ac2a-29721418c59c` value with the rule ID you want to ignore. In the example at the top of this section, the command to ignore the specific SCA rule ID is as follows: -`cycode ignore --scan-type sca --by-rule dc21bc6b-9f4f-46fb-9f92-e4327ea03f6b` +`cycode ignore -t sca --by-rule dc21bc6b-9f4f-46fb-9f92-e4327ea03f6b` In the example above, replace the `dc21bc6b-9f4f-46fb-9f92-e4327ea03f6b` value with the rule ID you want to ignore. -## Ignoring a Package +### Ignoring a Package -> :memo: **Note**
+> [!NOTE] > This option is only available to the SCA scans. To ignore a specific package in the SCA scans, you will need to use the `--by-package` flag in conjunction with the `-t, --scan-type` flag (you must specify the `sca` scan type). This will ignore the given package, using the `{{package_name}}@{{package_version}}` formatting, from all future scans. Use the following command to add a package and version to be ignored: @@ -676,6 +1203,160 @@ In the example below, the command to ignore a specific SCA package is as follows In the example above, replace `pyyaml` with package name and `5.3.1` with the package version you want to ignore. +### Ignoring via a config file + +The applied ignoring rules are stored in the configuration file called `config.yaml`. +This file could be easily shared between developers or even committed to remote Git. +These files are always located in the `.cycode` folder. +The folder starts with a dot (.), and you should enable the displaying of hidden files to see it. + +#### Path of the config files + +By default, all `cycode ignore` commands save the ignoring rule to the current directory from which CLI has been run. + +Example: running ignoring CLI command from `/Users/name/projects/backend` will create `config.yaml` in `/Users/name/projects/backend/.cycode` + +```shell +➜ backend pwd +/Users/name/projects/backend +➜ backend cycode ignore --by-value test-value +➜ backend tree -a +. +└── .cycode + └── config.yaml + +2 directories, 1 file +``` + +The second option is to save ignoring rules to the global configuration files. +The path of the global config is `~/.cycode/config.yaml`, +where `~` means user\`s home directory, for example, `/Users/name` on macOS. + +Saving to the global space could be performed with the `-g` flag of the `cycode ignore` command. +For example: `cycode ignore -g --by-value test-value`. + +#### Proper working directory + +It is incredibly important to place the `.cycode` folder and run CLI from the same place. +You should double-check it when working with different environments like CI/CD (GitHub Actions, Jenkins, etc.). + +You can commit the `.cycode` folder to the root of your repository. In this scenario, you must run CLI scans from the repository root. If that doesn't fit your requirements, you could temporarily copy the `.cycode` folder to wherever you want and perform a CLI scan from this folder. + +#### Structure ignoring rules in the config + +It's important to understand how CLI stores ignored rules to be able to read these configuration files or even modify them without CLI. + +The abstract YAML structure: +```yaml +exclusions: + {scanTypeName}: + {ignoringType}: + - someIgnoringValue1 + - someIgnoringValue2 +``` + +Possible values of `scanTypeName`: `iac`, `sca`, `sast`, `secret`. + +Possible values of `ignoringType`: `paths`, `values`, `rules`, `packages`, `shas`, `cves`. + +> [!WARNING] +> Values for "ignore by value" are not stored as plain text! +> CLI stores sha256 hashes of the values instead. +> You should put hashes of the string when modifying the configuration file by hand. + +Example of real `config.yaml`: +```yaml +exclusions: + iac: + rules: + - bdaa88e2-5e7c-46ff-ac2a-29721418c59c + sca: + packages: + - pyyaml@5.3.1 + secret: + paths: + - /Users/name/projects/build + rules: + - ce3a4de0-9dfc-448b-a004-c538cf8b4710 + shas: + - a44081db3296c84b82d12a35c446a3cba19411dddfa0380134c75f7b3973bff0 + values: + - a665a45920422f9d417e4867efdc4fb8a04a1f3fff1fa07e998e86f7f7a27ae3 + - 60303ae22b998861bce3b28f33eec1be758a213c86c93c076dbe9f558c11c752 +``` + +# Report Command + +## Generating SBOM Report + +A software bill of materials (SBOM) is an inventory of all constituent components and software dependencies involved in the development and delivery of an application. +Using this command, you can create an SBOM report for your local project or for your repository URI. + +The following options are available for use with this command: + +| Option | Description | Required | Default | +|----------------------------------------------------|--------------------------------|----------|-------------------------------------------------------| +| `-f, --format [spdx-2.2\|spdx-2.3\|cyclonedx-1.4]` | SBOM format | Yes | | +| `-o, --output-format [JSON]` | Specify the output file format | No | json | +| `--output-file PATH` | Output file | No | autogenerated filename saved to the current directory | +| `--include-vulnerabilities` | Include vulnerabilities | No | False | +| `--include-dev-dependencies` | Include dev dependencies | No | False | + +The following commands are available for use with this command: + +| Command | Description | +|------------------|-----------------------------------------------------------------| +| `path` | Generate SBOM report for provided path in the command | +| `repository-url` | Generate SBOM report for provided repository URI in the command | + +### Repository + +To create an SBOM report for a repository URI:\ +`cycode report sbom --format --include-vulnerabilities --include-dev-dependencies --output-file repository_url ` + +For example:\ +`cycode report sbom --format spdx-2.3 --include-vulnerabilities --include-dev-dependencies repository_url https://github.com/cycodehq/cycode-cli.git` + +### Local Project + +To create an SBOM report for a path:\ +`cycode report sbom --format --include-vulnerabilities --include-dev-dependencies --output-file path ` + +For example:\ +`cycode report sbom --format spdx-2.3 --include-vulnerabilities --include-dev-dependencies path /path/to/local/project` + +The `path` subcommand supports the following additional options: + +| Option | Description | +|-----------------------------|-------------------------------------------------------------------------------------------------------------------------------------| +| `--no-restore` | Skip lockfile restoration and scan direct dependencies only. See [Lock Restore Option](#lock-restore-option) for details. | +| `--gradle-all-sub-projects` | Run the Gradle restore command for all sub-projects (use from the root of a multi-project Gradle build). | +| `--maven-settings-file` | For Maven only, allows using a custom [settings.xml](https://maven.apache.org/settings.html) file when building the dependency tree. | + +# Import Command + +## Importing SBOM + +A software bill of materials (SBOM) is an inventory of all constituent components and software dependencies involved in the development and delivery of an application. +Using this command, you can import an SBOM file from your file system into Cycode. + +The following options are available for use with this command: + +| Option | Description | Required | Default | +|----------------------------------------------------|--------------------------------------------|----------|-------------------------------------------------------| +| `-n, --name TEXT` | Display name of the SBOM | Yes | | +| `-v, --vendor TEXT` | Name of the entity that provided the SBOM | Yes | | +| `-l, --label TEXT` | Attach label to the SBOM | No | | +| `-o, --owner TEXT` | Email address of the Cycode user that serves as point of contact for this SBOM | No | | +| `-b, --business-impact [High \| Medium \| Low]` | Business Impact | No | Medium | + +For example:\ +`cycode import sbom --name example-sbom --vendor cycode -label tag1 -label tag2 --owner example@cycode.com /path/to/local/project` + +# Scan Logs + +All CLI scans are logged in Cycode. The logs can be found under Settings > CLI Logs. + # Syntax Help You may add the `--help` argument to any command at any time to see a help message that will display available options and their syntax. @@ -692,10 +1373,18 @@ To see the options available for a specific type of scan, enter: `cycode scan {{option}} --help` -For example, to see options available for a Path Scan, you would simply enter: +For example, to see options available for a Path Scan, you would enter: `cycode scan path --help` To see the options available for the ignore scan function, use this command: `cycode ignore --help` + +To see the options available for a report, use this command: + +`cycode report --help` + +To see the options available for a specific type of report, enter: + +`cycode scan {{option}} --help` diff --git a/cycode/__init__.py b/cycode/__init__.py index 9d89bffd..4ce71ef1 100644 --- a/cycode/__init__.py +++ b/cycode/__init__.py @@ -1 +1 @@ -__version__ = '0.0.0' # DON'T TOUCH. Placeholder. Will be filled automatically on poetry build from Git Tag +__version__ = '0.0.0' # DON'T TOUCH. Placeholder. Will be filled automatically on poetry build from Git Tag diff --git a/cycode/__main__.py b/cycode/__main__.py new file mode 100644 index 00000000..7ad8ef7e --- /dev/null +++ b/cycode/__main__.py @@ -0,0 +1,4 @@ +from cycode.cli.consts import PROGRAM_NAME +from cycode.cli.main import app + +app(prog_name=PROGRAM_NAME) diff --git a/cycode/cli/app.py b/cycode/cli/app.py new file mode 100644 index 00000000..41391f99 --- /dev/null +++ b/cycode/cli/app.py @@ -0,0 +1,171 @@ +import logging +import sys +from typing import Annotated, Optional + +import typer +from typer import rich_utils +from typer._completion_classes import completion_init +from typer._completion_shared import Shells +from typer.completion import install_callback, show_callback + +from cycode import __version__ +from cycode.cli.apps import ai_guardrails, ai_remediation, auth, configure, ignore, report, report_import, scan, status + +if sys.version_info >= (3, 10): + from cycode.cli.apps import mcp + +from cycode.cli.cli_types import OutputTypeOption +from cycode.cli.consts import CLI_CONTEXT_SETTINGS +from cycode.cli.printers import ConsolePrinter +from cycode.cli.user_settings.configuration_manager import ConfigurationManager +from cycode.cli.utils.progress_bar import SCAN_PROGRESS_BAR_SECTIONS, get_progress_bar +from cycode.cli.utils.version_checker import version_checker +from cycode.cyclient.cycode_client_base import CycodeClientBase +from cycode.cyclient.models import UserAgentOptionScheme +from cycode.logger import set_logging_level + +# By default, it uses dim style which is hard to read with the combination of color from RICH_HELP +rich_utils.STYLE_ERRORS_SUGGESTION = 'bold' +# By default, it uses blue color which is too dark for some terminals +rich_utils.RICH_HELP = "Try [cyan]'{command_path} {help_option}'[/] for help." + +completion_init() # DO NOT TOUCH; this is required for the completion to work properly + +_cycode_cli_docs = 'https://github.com/cycodehq/cycode-cli/blob/main/README.md' +_cycode_cli_epilog = f'[bold]Documentation:[/] [link={_cycode_cli_docs}]{_cycode_cli_docs}[/link]' + +app = typer.Typer( + pretty_exceptions_show_locals=False, + pretty_exceptions_short=True, + context_settings=CLI_CONTEXT_SETTINGS, + epilog=_cycode_cli_epilog, + rich_markup_mode='rich', + no_args_is_help=True, + add_completion=False, # we add it manually to control the rich help panel +) + +app.add_typer(ai_guardrails.app) +app.add_typer(ai_remediation.app) +app.add_typer(auth.app) +app.add_typer(configure.app) +app.add_typer(ignore.app) +app.add_typer(report.app) +app.add_typer(report_import.app) +app.add_typer(scan.app) +app.add_typer(status.app) +if sys.version_info >= (3, 10): + app.add_typer(mcp.app) + + +def check_latest_version_on_close(ctx: typer.Context) -> None: + output = ctx.obj.get('output') + # don't print anything if the output is JSON + if output == OutputTypeOption.JSON: + return + + # we always want to check the latest version for "version" and "status" commands + should_use_cache = ctx.invoked_subcommand not in {'version', 'status'} + version_checker.check_and_notify_update(current_version=__version__, use_cache=should_use_cache) + + +def export_if_needed_on_close(ctx: typer.Context) -> None: + scan_finalized = ctx.obj.get('scan_finalized') + printer = ctx.obj.get('console_printer') + if scan_finalized and printer.is_recording: + printer.export() + + +_AUTH_RICH_HELP_PANEL = 'Authentication options' +_COMPLETION_RICH_HELP_PANEL = 'Completion options' + + +@app.callback() +def app_callback( + ctx: typer.Context, + verbose: Annotated[bool, typer.Option('--verbose', '-v', help='Show detailed logs.')] = False, + no_progress_meter: Annotated[ + bool, typer.Option('--no-progress-meter', help='Do not show the progress meter.') + ] = False, + no_update_notifier: Annotated[ + bool, typer.Option('--no-update-notifier', help='Do not check CLI for updates.') + ] = False, + output: Annotated[ + OutputTypeOption, typer.Option('--output', '-o', case_sensitive=False, help='Specify the output type.') + ] = OutputTypeOption.RICH, + user_agent: Annotated[ + Optional[str], + typer.Option(hidden=True, help='Characteristic JSON object that lets servers identify the application.'), + ] = None, + client_secret: Annotated[ + Optional[str], + typer.Option( + help='Specify a Cycode client secret for this specific scan execution.', + rich_help_panel=_AUTH_RICH_HELP_PANEL, + ), + ] = None, + client_id: Annotated[ + Optional[str], + typer.Option( + help='Specify a Cycode client ID for this specific scan execution.', + rich_help_panel=_AUTH_RICH_HELP_PANEL, + ), + ] = None, + id_token: Annotated[ + Optional[str], + typer.Option( + help='Specify a Cycode OIDC ID token for this specific scan execution.', + rich_help_panel=_AUTH_RICH_HELP_PANEL, + ), + ] = None, + _: Annotated[ + Optional[bool], + typer.Option( + '--install-completion', + callback=install_callback, + is_eager=True, + expose_value=False, + help='Install completion for the current shell.', + rich_help_panel=_COMPLETION_RICH_HELP_PANEL, + ), + ] = False, + __: Annotated[ + Shells, # the choice is required for Homebrew to be able to install the completion + typer.Option( + '--show-completion', + callback=show_callback, + is_eager=True, + expose_value=False, + show_default=False, + help='Show completion for the specified shell, to copy it or customize the installation.', + rich_help_panel=_COMPLETION_RICH_HELP_PANEL, + ), + ] = None, +) -> None: + """[bold cyan]Cycode CLI - Command Line Interface for Cycode.[/]""" + ctx.ensure_object(dict) + configuration_manager = ConfigurationManager() + + verbose = verbose or configuration_manager.get_verbose_flag() + ctx.obj['verbose'] = verbose + if verbose: + set_logging_level(logging.DEBUG) + + ctx.obj['output'] = output + if output == OutputTypeOption.JSON: + no_progress_meter = True + + ctx.obj['client_id'] = client_id + ctx.obj['client_secret'] = client_secret + ctx.obj['id_token'] = id_token + + ctx.obj['progress_bar'] = get_progress_bar(hidden=no_progress_meter, sections=SCAN_PROGRESS_BAR_SECTIONS) + + ctx.obj['console_printer'] = ConsolePrinter(ctx) + ctx.call_on_close(lambda: export_if_needed_on_close(ctx)) + + if user_agent: + user_agent_option = UserAgentOptionScheme().loads(user_agent) + CycodeClientBase.enrich_user_agent(user_agent_option.user_agent_suffix) + + if not no_update_notifier: + ctx.call_on_close(lambda: check_latest_version_on_close(ctx)) diff --git a/cycode/cli/auth/__init__.py b/cycode/cli/apps/__init__.py similarity index 100% rename from cycode/cli/auth/__init__.py rename to cycode/cli/apps/__init__.py diff --git a/cycode/cli/apps/ai_guardrails/__init__.py b/cycode/cli/apps/ai_guardrails/__init__.py new file mode 100644 index 00000000..f8486ed4 --- /dev/null +++ b/cycode/cli/apps/ai_guardrails/__init__.py @@ -0,0 +1,19 @@ +import typer + +from cycode.cli.apps.ai_guardrails.install_command import install_command +from cycode.cli.apps.ai_guardrails.scan.scan_command import scan_command +from cycode.cli.apps.ai_guardrails.status_command import status_command +from cycode.cli.apps.ai_guardrails.uninstall_command import uninstall_command + +app = typer.Typer(name='ai-guardrails', no_args_is_help=True, hidden=True) + +app.command(hidden=True, name='install', short_help='Install AI guardrails hooks for supported IDEs.')(install_command) +app.command(hidden=True, name='uninstall', short_help='Remove AI guardrails hooks from supported IDEs.')( + uninstall_command +) +app.command(hidden=True, name='status', short_help='Show AI guardrails hook installation status.')(status_command) +app.command( + hidden=True, + name='scan', + short_help='Scan content from AI IDE hooks for secrets (reads JSON from stdin).', +)(scan_command) diff --git a/cycode/cli/apps/ai_guardrails/command_utils.py b/cycode/cli/apps/ai_guardrails/command_utils.py new file mode 100644 index 00000000..edc3104a --- /dev/null +++ b/cycode/cli/apps/ai_guardrails/command_utils.py @@ -0,0 +1,68 @@ +"""Common utilities for AI guardrails commands.""" + +import os +from pathlib import Path +from typing import Optional + +import typer +from rich.console import Console + +from cycode.cli.apps.ai_guardrails.consts import AIIDEType + +console = Console() + + +def validate_and_parse_ide(ide: str) -> Optional[AIIDEType]: + """Validate IDE parameter, returning None for 'all'. + + Args: + ide: IDE name string (e.g., 'cursor', 'claude-code', 'all') + + Returns: + AIIDEType enum value, or None if 'all' was specified + + Raises: + typer.Exit: If IDE is invalid + """ + if ide.lower() == 'all': + return None + try: + return AIIDEType(ide.lower()) + except ValueError: + valid_ides = ', '.join([ide_type.value for ide_type in AIIDEType]) + console.print( + f'[red]Error:[/] Invalid IDE "{ide}". Supported IDEs: {valid_ides}, all', + style='bold red', + ) + raise typer.Exit(1) from None + + +def validate_scope(scope: str, allowed_scopes: tuple[str, ...] = ('user', 'repo')) -> None: + """Validate scope parameter. + + Args: + scope: Scope string to validate + allowed_scopes: Tuple of allowed scope values + + Raises: + typer.Exit: If scope is invalid + """ + if scope not in allowed_scopes: + scopes_list = ', '.join(f'"{s}"' for s in allowed_scopes) + console.print(f'[red]Error:[/] Invalid scope. Use {scopes_list}.', style='bold red') + raise typer.Exit(1) + + +def resolve_repo_path(scope: str, repo_path: Optional[Path]) -> Optional[Path]: + """Resolve repository path, defaulting to current directory for repo scope. + + Args: + scope: The command scope ('user' or 'repo') + repo_path: Provided repo path or None + + Returns: + Resolved Path for repo scope, None for user scope + """ + if scope == 'repo' and repo_path is None: + return Path(os.getcwd()) + return repo_path diff --git a/cycode/cli/apps/ai_guardrails/consts.py b/cycode/cli/apps/ai_guardrails/consts.py new file mode 100644 index 00000000..8714ec10 --- /dev/null +++ b/cycode/cli/apps/ai_guardrails/consts.py @@ -0,0 +1,133 @@ +"""Constants for AI guardrails hooks management. + +Currently supports: +- Cursor +- Claude Code +""" + +import platform +from enum import Enum +from pathlib import Path +from typing import NamedTuple + + +class AIIDEType(str, Enum): + """Supported AI IDE types.""" + + CURSOR = 'cursor' + CLAUDE_CODE = 'claude-code' + + +class PolicyMode(str, Enum): + """Policy enforcement mode for global mode and per-feature actions.""" + + BLOCK = 'block' + WARN = 'warn' + + +class IDEConfig(NamedTuple): + """Configuration for an AI IDE.""" + + name: str + hooks_dir: Path + repo_hooks_subdir: str # Subdirectory in repo for hooks (e.g., '.cursor') + hooks_file_name: str + hook_events: list[str] # List of supported hook event names for this IDE + + +def _get_cursor_hooks_dir() -> Path: + """Get Cursor hooks directory based on platform.""" + if platform.system() == 'Darwin': + return Path.home() / '.cursor' + if platform.system() == 'Windows': + return Path.home() / 'AppData' / 'Roaming' / 'Cursor' + # Linux + return Path.home() / '.config' / 'Cursor' + + +def _get_claude_code_hooks_dir() -> Path: + """Get Claude Code hooks directory. + + Claude Code uses ~/.claude on all platforms. + """ + return Path.home() / '.claude' + + +# IDE-specific configurations +IDE_CONFIGS: dict[AIIDEType, IDEConfig] = { + AIIDEType.CURSOR: IDEConfig( + name='Cursor', + hooks_dir=_get_cursor_hooks_dir(), + repo_hooks_subdir='.cursor', + hooks_file_name='hooks.json', + hook_events=['beforeSubmitPrompt', 'beforeReadFile', 'beforeMCPExecution'], + ), + AIIDEType.CLAUDE_CODE: IDEConfig( + name='Claude Code', + hooks_dir=_get_claude_code_hooks_dir(), + repo_hooks_subdir='.claude', + hooks_file_name='settings.json', + hook_events=['UserPromptSubmit', 'PreToolUse:Read', 'PreToolUse:mcp'], + ), +} + +# Default IDE +DEFAULT_IDE = AIIDEType.CURSOR + +# Command used in hooks +CYCODE_SCAN_PROMPT_COMMAND = 'cycode ai-guardrails scan' + + +def _get_cursor_hooks_config() -> dict: + """Get Cursor-specific hooks configuration.""" + config = IDE_CONFIGS[AIIDEType.CURSOR] + hooks = {event: [{'command': CYCODE_SCAN_PROMPT_COMMAND}] for event in config.hook_events} + + return { + 'version': 1, + 'hooks': hooks, + } + + +def _get_claude_code_hooks_config() -> dict: + """Get Claude Code-specific hooks configuration. + + Claude Code uses a different hook format with nested structure: + - hooks are arrays of objects with 'hooks' containing command arrays + - PreToolUse uses 'matcher' field to specify which tools to intercept + """ + command = f'{CYCODE_SCAN_PROMPT_COMMAND} --ide claude-code' + + return { + 'hooks': { + 'UserPromptSubmit': [ + { + 'hooks': [{'type': 'command', 'command': command}], + } + ], + 'PreToolUse': [ + { + 'matcher': 'Read', + 'hooks': [{'type': 'command', 'command': command}], + }, + { + 'matcher': 'mcp__.*', + 'hooks': [{'type': 'command', 'command': command}], + }, + ], + }, + } + + +def get_hooks_config(ide: AIIDEType) -> dict: + """Get the hooks configuration for a specific IDE. + + Args: + ide: The AI IDE type + + Returns: + Dict with hooks configuration for the specified IDE + """ + if ide == AIIDEType.CLAUDE_CODE: + return _get_claude_code_hooks_config() + return _get_cursor_hooks_config() diff --git a/cycode/cli/apps/ai_guardrails/hooks_manager.py b/cycode/cli/apps/ai_guardrails/hooks_manager.py new file mode 100644 index 00000000..b8d43c43 --- /dev/null +++ b/cycode/cli/apps/ai_guardrails/hooks_manager.py @@ -0,0 +1,226 @@ +""" +Hooks manager for AI guardrails. + +Handles installation, removal, and status checking of AI IDE hooks. +Supports multiple IDEs: Cursor, Claude Code (future). +""" + +import json +from pathlib import Path +from typing import Optional + +from cycode.cli.apps.ai_guardrails.consts import ( + CYCODE_SCAN_PROMPT_COMMAND, + DEFAULT_IDE, + IDE_CONFIGS, + AIIDEType, + get_hooks_config, +) +from cycode.logger import get_logger + +logger = get_logger('AI Guardrails Hooks') + + +def get_hooks_path(scope: str, repo_path: Optional[Path] = None, ide: AIIDEType = DEFAULT_IDE) -> Path: + """Get the hooks.json path for the given scope and IDE. + + Args: + scope: 'user' for user-level hooks, 'repo' for repository-level hooks + repo_path: Repository path (required if scope is 'repo') + ide: The AI IDE type (default: Cursor) + """ + config = IDE_CONFIGS[ide] + if scope == 'repo' and repo_path: + return repo_path / config.repo_hooks_subdir / config.hooks_file_name + return config.hooks_dir / config.hooks_file_name + + +def load_hooks_file(hooks_path: Path) -> Optional[dict]: + """Load existing hooks.json file.""" + if not hooks_path.exists(): + return None + try: + content = hooks_path.read_text(encoding='utf-8') + return json.loads(content) + except Exception as e: + logger.debug('Failed to load hooks file', exc_info=e) + return None + + +def save_hooks_file(hooks_path: Path, hooks_config: dict) -> bool: + """Save hooks.json file.""" + try: + hooks_path.parent.mkdir(parents=True, exist_ok=True) + hooks_path.write_text(json.dumps(hooks_config, indent=2), encoding='utf-8') + return True + except Exception as e: + logger.error('Failed to save hooks file', exc_info=e) + return False + + +def is_cycode_hook_entry(entry: dict) -> bool: + """Check if a hook entry is from cycode-cli. + + Handles both Cursor format (flat) and Claude Code format (nested). + + Cursor format: {"command": "cycode ai-guardrails scan"} + Claude Code format: {"hooks": [{"type": "command", "command": "cycode ai-guardrails scan --ide claude-code"}]} + """ + # Check Cursor format (flat command) + command = entry.get('command', '') + if CYCODE_SCAN_PROMPT_COMMAND in command: + return True + + # Check Claude Code format (nested hooks array) + hooks = entry.get('hooks', []) + for hook in hooks: + if isinstance(hook, dict): + hook_command = hook.get('command', '') + if CYCODE_SCAN_PROMPT_COMMAND in hook_command: + return True + + return False + + +def install_hooks( + scope: str = 'user', repo_path: Optional[Path] = None, ide: AIIDEType = DEFAULT_IDE +) -> tuple[bool, str]: + """ + Install Cycode AI guardrails hooks. + + Args: + scope: 'user' for user-level hooks, 'repo' for repository-level hooks + repo_path: Repository path (required if scope is 'repo') + ide: The AI IDE type (default: Cursor) + + Returns: + Tuple of (success, message) + """ + hooks_path = get_hooks_path(scope, repo_path, ide) + + # Load existing hooks or create new + existing = load_hooks_file(hooks_path) or {'version': 1, 'hooks': {}} + existing.setdefault('version', 1) + existing.setdefault('hooks', {}) + + # Get IDE-specific hooks configuration + hooks_config = get_hooks_config(ide) + + # Add/update Cycode hooks + for event, entries in hooks_config['hooks'].items(): + existing['hooks'].setdefault(event, []) + + # Remove any existing Cycode entries for this event + existing['hooks'][event] = [e for e in existing['hooks'][event] if not is_cycode_hook_entry(e)] + + # Add new Cycode entries + for entry in entries: + existing['hooks'][event].append(entry) + + # Save + if save_hooks_file(hooks_path, existing): + return True, f'AI guardrails hooks installed: {hooks_path}' + return False, f'Failed to install hooks to {hooks_path}' + + +def uninstall_hooks( + scope: str = 'user', repo_path: Optional[Path] = None, ide: AIIDEType = DEFAULT_IDE +) -> tuple[bool, str]: + """ + Remove Cycode AI guardrails hooks. + + Args: + scope: 'user' for user-level hooks, 'repo' for repository-level hooks + repo_path: Repository path (required if scope is 'repo') + ide: The AI IDE type (default: Cursor) + + Returns: + Tuple of (success, message) + """ + hooks_path = get_hooks_path(scope, repo_path, ide) + + existing = load_hooks_file(hooks_path) + if existing is None: + return True, f'No hooks file found at {hooks_path}' + + # Remove Cycode entries from all events + modified = False + for event in list(existing.get('hooks', {}).keys()): + original_count = len(existing['hooks'][event]) + existing['hooks'][event] = [e for e in existing['hooks'][event] if not is_cycode_hook_entry(e)] + if len(existing['hooks'][event]) != original_count: + modified = True + # Remove empty event lists + if not existing['hooks'][event]: + del existing['hooks'][event] + + if not modified: + return True, 'No Cycode hooks found to remove' + + # Save or delete if empty + if not existing.get('hooks'): + try: + hooks_path.unlink() + return True, f'Removed hooks file: {hooks_path}' + except Exception as e: + logger.debug('Failed to delete hooks file', exc_info=e) + return False, f'Failed to remove hooks file: {hooks_path}' + + if save_hooks_file(hooks_path, existing): + return True, f'Cycode hooks removed from: {hooks_path}' + return False, f'Failed to update hooks file: {hooks_path}' + + +def get_hooks_status(scope: str = 'user', repo_path: Optional[Path] = None, ide: AIIDEType = DEFAULT_IDE) -> dict: + """ + Get the status of AI guardrails hooks. + + Args: + scope: 'user' for user-level hooks, 'repo' for repository-level hooks + repo_path: Repository path (required if scope is 'repo') + ide: The AI IDE type (default: Cursor) + + Returns: + Dict with status information + """ + hooks_path = get_hooks_path(scope, repo_path, ide) + + status = { + 'scope': scope, + 'ide': ide.value, + 'ide_name': IDE_CONFIGS[ide].name, + 'hooks_path': str(hooks_path), + 'file_exists': hooks_path.exists(), + 'cycode_installed': False, + 'hooks': {}, + } + + existing = load_hooks_file(hooks_path) + if existing is None: + return status + + # Check each hook event for this IDE + ide_config = IDE_CONFIGS[ide] + has_cycode_hooks = False + for event in ide_config.hook_events: + # Handle event:matcher format + if ':' in event: + actual_event, matcher_prefix = event.split(':', 1) + all_entries = existing.get('hooks', {}).get(actual_event, []) + # Filter entries by matcher + entries = [e for e in all_entries if e.get('matcher', '').startswith(matcher_prefix)] + else: + entries = existing.get('hooks', {}).get(event, []) + + cycode_entries = [e for e in entries if is_cycode_hook_entry(e)] + if cycode_entries: + has_cycode_hooks = True + status['hooks'][event] = { + 'total_entries': len(entries), + 'cycode_entries': len(cycode_entries), + 'enabled': len(cycode_entries) > 0, + } + + status['cycode_installed'] = has_cycode_hooks + + return status diff --git a/cycode/cli/apps/ai_guardrails/install_command.py b/cycode/cli/apps/ai_guardrails/install_command.py new file mode 100644 index 00000000..a72d5d4c --- /dev/null +++ b/cycode/cli/apps/ai_guardrails/install_command.py @@ -0,0 +1,94 @@ +"""Install command for AI guardrails hooks.""" + +from pathlib import Path +from typing import Annotated, Optional + +import typer + +from cycode.cli.apps.ai_guardrails.command_utils import ( + console, + resolve_repo_path, + validate_and_parse_ide, + validate_scope, +) +from cycode.cli.apps.ai_guardrails.consts import IDE_CONFIGS, AIIDEType +from cycode.cli.apps.ai_guardrails.hooks_manager import install_hooks + + +def install_command( + ctx: typer.Context, + scope: Annotated[ + str, + typer.Option( + '--scope', + '-s', + help='Installation scope: "user" for all projects, "repo" for current repository only.', + ), + ] = 'user', + ide: Annotated[ + str, + typer.Option( + '--ide', + help='IDE to install hooks for (e.g., "cursor", "claude-code", or "all" for all IDEs). Defaults to cursor.', + ), + ] = AIIDEType.CURSOR.value, + repo_path: Annotated[ + Optional[Path], + typer.Option( + '--repo-path', + help='Repository path for repo-scoped installation (defaults to current directory).', + exists=True, + file_okay=False, + dir_okay=True, + resolve_path=True, + ), + ] = None, +) -> None: + """Install AI guardrails hooks for supported IDEs. + + This command configures the specified IDE to use Cycode for scanning prompts, file reads, + and MCP tool calls for secrets before they are sent to AI models. + + Examples: + cycode ai-guardrails install # Install for all projects (user scope) + cycode ai-guardrails install --scope repo # Install for current repo only + cycode ai-guardrails install --ide cursor # Install for Cursor IDE + cycode ai-guardrails install --ide all # Install for all supported IDEs + cycode ai-guardrails install --scope repo --repo-path /path/to/repo + """ + # Validate inputs + validate_scope(scope) + repo_path = resolve_repo_path(scope, repo_path) + ide_type = validate_and_parse_ide(ide) + + ides_to_install: list[AIIDEType] = list(AIIDEType) if ide_type is None else [ide_type] + + results: list[tuple[str, bool, str]] = [] + for current_ide in ides_to_install: + ide_name = IDE_CONFIGS[current_ide].name + success, message = install_hooks(scope, repo_path, ide=current_ide) + results.append((ide_name, success, message)) + + # Report results for each IDE + any_success = False + all_success = True + for _ide_name, success, message in results: + if success: + console.print(f'[green]✓[/] {message}') + any_success = True + else: + console.print(f'[red]✗[/] {message}', style='bold red') + all_success = False + + if any_success: + console.print() + console.print('[bold]Next steps:[/]') + successful_ides = [name for name, success, _ in results if success] + ide_list = ', '.join(successful_ides) + console.print(f'1. Restart {ide_list} to activate the hooks') + console.print('2. (Optional) Customize policy in ~/.cycode/ai-guardrails.yaml') + console.print() + console.print('[dim]The hooks will scan prompts, file reads, and MCP tool calls for secrets.[/]') + + if not all_success: + raise typer.Exit(1) diff --git a/cycode/cli/apps/ai_guardrails/scan/__init__.py b/cycode/cli/apps/ai_guardrails/scan/__init__.py new file mode 100644 index 00000000..47349e78 --- /dev/null +++ b/cycode/cli/apps/ai_guardrails/scan/__init__.py @@ -0,0 +1 @@ +# Prompt scan command for AI guardrails (hooks) diff --git a/cycode/cli/apps/ai_guardrails/scan/consts.py b/cycode/cli/apps/ai_guardrails/scan/consts.py new file mode 100644 index 00000000..007892a8 --- /dev/null +++ b/cycode/cli/apps/ai_guardrails/scan/consts.py @@ -0,0 +1,48 @@ +""" +Constants and default configuration for AI guardrails. + +These defaults can be overridden by: +1. User-level config: ~/.cycode/ai-guardrails.yaml +2. Repo-level config: /.cycode/ai-guardrails.yaml +""" + +# Policy file name +POLICY_FILE_NAME = 'ai-guardrails.yaml' + +# Default policy configuration +DEFAULT_POLICY = { + 'version': 1, + 'mode': 'block', # block | warn + 'fail_open': True, # allow if scan fails/timeouts + 'secrets': { + 'scan_type': 'secret', + 'timeout_ms': 30000, + 'max_bytes': 200000, + }, + 'prompt': { + 'enabled': True, + 'action': 'block', + }, + 'file_read': { + 'enabled': True, + 'action': 'block', + 'deny_globs': [ + '.env', + '.env.*', + '*.pem', + '*.p12', + '*.key', + '.aws/**', + '.ssh/**', + '*kubeconfig*', + '.npmrc', + '.netrc', + ], + 'scan_content': True, + }, + 'mcp': { + 'enabled': True, + 'action': 'block', + 'scan_arguments': True, + }, +} diff --git a/cycode/cli/apps/ai_guardrails/scan/handlers.py b/cycode/cli/apps/ai_guardrails/scan/handlers.py new file mode 100644 index 00000000..2a762a8d --- /dev/null +++ b/cycode/cli/apps/ai_guardrails/scan/handlers.py @@ -0,0 +1,361 @@ +""" +Hook handlers for AI IDE events. + +Each handler receives a unified payload from an IDE, applies policy rules, +and returns a response that either allows or blocks the action. +""" + +import json +import os +from multiprocessing.pool import ThreadPool +from multiprocessing.pool import TimeoutError as PoolTimeoutError +from typing import Callable, Optional + +import typer + +from cycode.cli.apps.ai_guardrails.consts import PolicyMode +from cycode.cli.apps.ai_guardrails.scan.payload import AIHookPayload +from cycode.cli.apps.ai_guardrails.scan.policy import get_policy_value +from cycode.cli.apps.ai_guardrails.scan.response_builders import get_response_builder +from cycode.cli.apps.ai_guardrails.scan.types import AiHookEventType, AIHookOutcome, BlockReason +from cycode.cli.apps.ai_guardrails.scan.utils import is_denied_path, truncate_utf8 +from cycode.cli.apps.scan.code_scanner import _get_scan_documents_thread_func +from cycode.cli.apps.scan.scan_parameters import get_scan_parameters +from cycode.cli.cli_types import ScanTypeOption, SeverityOption +from cycode.cli.models import Document +from cycode.cli.utils.progress_bar import DummyProgressBar, ScanProgressBarSection +from cycode.cli.utils.scan_utils import build_violation_summary +from cycode.logger import get_logger + +logger = get_logger('AI Guardrails') + + +def handle_before_submit_prompt(ctx: typer.Context, payload: AIHookPayload, policy: dict) -> dict: + """ + Handle beforeSubmitPrompt hook. + + Scans prompt text for secrets before it's sent to the AI model. + Returns {"continue": False} to block, {"continue": True} to allow. + """ + ai_client = ctx.obj['ai_security_client'] + ide = payload.ide_provider + response_builder = get_response_builder(ide) + + prompt_config = get_policy_value(policy, 'prompt', default={}) + ai_client.create_conversation(payload) + if not get_policy_value(prompt_config, 'enabled', default=True): + ai_client.create_event(payload, AiHookEventType.PROMPT, AIHookOutcome.ALLOWED) + return response_builder.allow_prompt() + + mode = get_policy_value(policy, 'mode', default=PolicyMode.BLOCK) + prompt = payload.prompt or '' + max_bytes = get_policy_value(policy, 'secrets', 'max_bytes', default=200000) + timeout_ms = get_policy_value(policy, 'secrets', 'timeout_ms', default=30000) + clipped = truncate_utf8(prompt, max_bytes) + + scan_id = None + block_reason = None + outcome = AIHookOutcome.ALLOWED + error_message = None + + try: + violation_summary, scan_id = _scan_text_for_secrets(ctx, clipped, timeout_ms) + + if violation_summary: + block_reason = BlockReason.SECRETS_IN_PROMPT + action = get_policy_value(prompt_config, 'action', default=PolicyMode.BLOCK) + if action == PolicyMode.BLOCK and mode == PolicyMode.BLOCK: + outcome = AIHookOutcome.BLOCKED + user_message = f'{violation_summary}. Remove secrets before sending.' + return response_builder.deny_prompt(user_message) + outcome = AIHookOutcome.WARNED + return response_builder.allow_prompt() + except Exception as e: + outcome = ( + AIHookOutcome.ALLOWED if get_policy_value(policy, 'fail_open', default=True) else AIHookOutcome.BLOCKED + ) + block_reason = BlockReason.SCAN_FAILURE + error_message = str(e) + raise e + finally: + ai_client.create_event( + payload, + AiHookEventType.PROMPT, + outcome, + scan_id=scan_id, + block_reason=block_reason, + error_message=error_message, + ) + + +def handle_before_read_file(ctx: typer.Context, payload: AIHookPayload, policy: dict) -> dict: + """ + Handle beforeReadFile hook. + + Blocks sensitive files (via deny_globs) and scans file content for secrets. + Returns {"permission": "deny"} to block, {"permission": "allow"} to allow. + """ + ai_client = ctx.obj['ai_security_client'] + ide = payload.ide_provider + response_builder = get_response_builder(ide) + + file_read_config = get_policy_value(policy, 'file_read', default={}) + ai_client.create_conversation(payload) + if not get_policy_value(file_read_config, 'enabled', default=True): + ai_client.create_event(payload, AiHookEventType.FILE_READ, AIHookOutcome.ALLOWED) + return response_builder.allow_permission() + + mode = get_policy_value(policy, 'mode', default=PolicyMode.BLOCK) + file_path = payload.file_path or '' + action = get_policy_value(file_read_config, 'action', default=PolicyMode.BLOCK) + + scan_id = None + block_reason = None + outcome = AIHookOutcome.ALLOWED + error_message = None + + try: + # Check path-based denylist first + if is_denied_path(file_path, policy): + block_reason = BlockReason.SENSITIVE_PATH + if mode == PolicyMode.BLOCK and action == PolicyMode.BLOCK: + outcome = AIHookOutcome.BLOCKED + user_message = f'Cycode blocked sending {file_path} to the AI (sensitive path policy).' + return response_builder.deny_permission( + user_message, + 'This file path is classified as sensitive; do not read/send it to the model.', + ) + # Warn mode - ask user for permission + outcome = AIHookOutcome.WARNED + user_message = f'Cycode flagged {file_path} as sensitive. Allow reading?' + return response_builder.ask_permission( + user_message, + 'This file path is classified as sensitive; proceed with caution.', + ) + + # Scan file content if enabled + if get_policy_value(file_read_config, 'scan_content', default=True): + violation_summary, scan_id = _scan_path_for_secrets(ctx, file_path, policy) + if violation_summary: + block_reason = BlockReason.SECRETS_IN_FILE + if mode == PolicyMode.BLOCK and action == PolicyMode.BLOCK: + outcome = AIHookOutcome.BLOCKED + user_message = f'Cycode blocked reading {file_path}. {violation_summary}' + return response_builder.deny_permission( + user_message, + 'Secrets detected; do not send this file to the model.', + ) + # Warn mode - ask user for permission + outcome = AIHookOutcome.WARNED + user_message = f'Cycode detected secrets in {file_path}. {violation_summary}' + return response_builder.ask_permission( + user_message, + 'Possible secrets detected; proceed with caution.', + ) + return response_builder.allow_permission() + + return response_builder.allow_permission() + except Exception as e: + outcome = ( + AIHookOutcome.ALLOWED if get_policy_value(policy, 'fail_open', default=True) else AIHookOutcome.BLOCKED + ) + block_reason = BlockReason.SCAN_FAILURE + error_message = str(e) + raise e + finally: + ai_client.create_event( + payload, + AiHookEventType.FILE_READ, + outcome, + scan_id=scan_id, + block_reason=block_reason, + error_message=error_message, + file_path=payload.file_path, + ) + + +def handle_before_mcp_execution(ctx: typer.Context, payload: AIHookPayload, policy: dict) -> dict: + """ + Handle beforeMCPExecution hook. + + Scans tool arguments for secrets before MCP tool execution. + Returns {"permission": "deny"} to block, {"permission": "ask"} to warn, + {"permission": "allow"} to allow. + """ + ai_client = ctx.obj['ai_security_client'] + ide = payload.ide_provider + response_builder = get_response_builder(ide) + + mcp_config = get_policy_value(policy, 'mcp', default={}) + ai_client.create_conversation(payload) + if not get_policy_value(mcp_config, 'enabled', default=True): + ai_client.create_event(payload, AiHookEventType.MCP_EXECUTION, AIHookOutcome.ALLOWED) + return response_builder.allow_permission() + + mode = get_policy_value(policy, 'mode', default=PolicyMode.BLOCK) + tool = payload.mcp_tool_name or 'unknown' + args = payload.mcp_arguments or {} + args_text = args if isinstance(args, str) else json.dumps(args) + max_bytes = get_policy_value(policy, 'secrets', 'max_bytes', default=200000) + timeout_ms = get_policy_value(policy, 'secrets', 'timeout_ms', default=30000) + clipped = truncate_utf8(args_text, max_bytes) + action = get_policy_value(mcp_config, 'action', default=PolicyMode.BLOCK) + + scan_id = None + block_reason = None + outcome = AIHookOutcome.ALLOWED + error_message = None + + try: + if get_policy_value(mcp_config, 'scan_arguments', default=True): + violation_summary, scan_id = _scan_text_for_secrets(ctx, clipped, timeout_ms) + if violation_summary: + block_reason = BlockReason.SECRETS_IN_MCP_ARGS + if mode == PolicyMode.BLOCK and action == PolicyMode.BLOCK: + outcome = AIHookOutcome.BLOCKED + user_message = f'Cycode blocked MCP tool call "{tool}". {violation_summary}' + return response_builder.deny_permission( + user_message, + 'Do not pass secrets to tools. Use secret references (name/id) instead.', + ) + outcome = AIHookOutcome.WARNED + return response_builder.ask_permission( + f'{violation_summary} in MCP tool call "{tool}". Allow execution?', + 'Possible secrets detected in tool arguments; proceed with caution.', + ) + + return response_builder.allow_permission() + except Exception as e: + outcome = ( + AIHookOutcome.ALLOWED if get_policy_value(policy, 'fail_open', default=True) else AIHookOutcome.BLOCKED + ) + block_reason = BlockReason.SCAN_FAILURE + error_message = str(e) + raise e + finally: + ai_client.create_event( + payload, + AiHookEventType.MCP_EXECUTION, + outcome, + scan_id=scan_id, + block_reason=block_reason, + error_message=error_message, + ) + + +def get_handler_for_event(event_type: str) -> Optional[Callable[[typer.Context, AIHookPayload, dict], dict]]: + """Get the appropriate handler function for a canonical event type. + + Args: + event_type: Canonical event type string (from AiHookEventType enum) + + Returns: + Handler function or None if event type is not recognized + """ + handlers = { + AiHookEventType.PROMPT.value: handle_before_submit_prompt, + AiHookEventType.FILE_READ.value: handle_before_read_file, + AiHookEventType.MCP_EXECUTION.value: handle_before_mcp_execution, + } + return handlers.get(event_type) + + +def _setup_scan_context(ctx: typer.Context) -> typer.Context: + """Set up minimal context for scan_documents without progress bars or printing.""" + + # Set up minimal required context + ctx.obj['progress_bar'] = DummyProgressBar([ScanProgressBarSection]) + ctx.obj['sync'] = True # Synchronous scan + ctx.obj['scan_type'] = ScanTypeOption.SECRET # AI guardrails always scans for secrets + ctx.obj['severity_threshold'] = SeverityOption.INFO # Report all severities + + # Set command name for scan logic + ctx.info_name = 'ai_guardrails' + + return ctx + + +def _perform_scan( + ctx: typer.Context, documents: list[Document], scan_parameters: dict, timeout_seconds: float +) -> tuple[Optional[str], Optional[str]]: + """ + Perform a scan on documents and extract results. + + Returns tuple of (violation_summary, scan_id) if secrets found, (None, scan_id) if clean. + Raises exception if scan fails or times out (triggers fail_open policy). + """ + if not documents: + return None, None + + # Get the thread function for scanning + scan_batch_thread_func = _get_scan_documents_thread_func( + ctx, is_git_diff=False, is_commit_range=False, scan_parameters=scan_parameters + ) + + # Use ThreadPool.apply_async with timeout to abort if scan takes too long + # This uses the same ThreadPool mechanism as run_parallel_batched_scan but with timeout support + with ThreadPool(processes=1) as pool: + result = pool.apply_async(scan_batch_thread_func, (documents,)) + try: + scan_id, error, local_scan_result = result.get(timeout=timeout_seconds) + except PoolTimeoutError: + logger.debug('Scan timed out after %s seconds', timeout_seconds) + raise RuntimeError(f'Scan timed out after {timeout_seconds} seconds') from None + + # Check if scan failed - raise exception to trigger fail_open policy + if error: + raise RuntimeError(error.message) + + if not local_scan_result: + return None, None + + scan_id = local_scan_result.scan_id + + # Check if there are any detections + if local_scan_result.detections_count > 0: + violation_summary = build_violation_summary([local_scan_result]) + return violation_summary, scan_id + + return None, scan_id + + +def _scan_text_for_secrets(ctx: typer.Context, text: str, timeout_ms: int) -> tuple[Optional[str], Optional[str]]: + """ + Scan text content for secrets using Cycode CLI. + + Returns tuple of (violation_summary, scan_id) if secrets found, (None, scan_id) if clean. + Raises exception on error or timeout. + """ + if not text: + return None, None + + document = Document(path='prompt-content.txt', content=text, is_git_diff_format=False) + scan_ctx = _setup_scan_context(ctx) + timeout_seconds = timeout_ms / 1000.0 + return _perform_scan(scan_ctx, [document], get_scan_parameters(scan_ctx, None), timeout_seconds) + + +def _scan_path_for_secrets(ctx: typer.Context, file_path: str, policy: dict) -> tuple[Optional[str], Optional[str]]: + """ + Scan a file path for secrets. + + Returns tuple of (violation_summary, scan_id) if secrets found, (None, scan_id) if clean. + Raises exception on error or timeout. + """ + if not file_path or not os.path.exists(file_path): + return None, None + + with open(file_path, encoding='utf-8', errors='replace') as f: + content = f.read() + + # Truncate content based on policy max_bytes + max_bytes = get_policy_value(policy, 'secrets', 'max_bytes', default=200000) + content = truncate_utf8(content, max_bytes) + + # Get timeout from policy + timeout_ms = get_policy_value(policy, 'secrets', 'timeout_ms', default=30000) + timeout_seconds = timeout_ms / 1000.0 + + document = Document(path=os.path.basename(file_path), content=content, is_git_diff_format=False) + scan_ctx = _setup_scan_context(ctx) + return _perform_scan(scan_ctx, [document], get_scan_parameters(scan_ctx, (file_path,)), timeout_seconds) diff --git a/cycode/cli/apps/ai_guardrails/scan/payload.py b/cycode/cli/apps/ai_guardrails/scan/payload.py new file mode 100644 index 00000000..08e96f9a --- /dev/null +++ b/cycode/cli/apps/ai_guardrails/scan/payload.py @@ -0,0 +1,268 @@ +"""Unified payload object for AI hook events from different tools.""" + +import json +from collections.abc import Iterator +from dataclasses import dataclass +from pathlib import Path +from typing import Optional + +from cycode.cli.apps.ai_guardrails.consts import AIIDEType +from cycode.cli.apps.ai_guardrails.scan.types import ( + CLAUDE_CODE_EVENT_MAPPING, + CLAUDE_CODE_EVENT_NAMES, + CURSOR_EVENT_MAPPING, + CURSOR_EVENT_NAMES, + AiHookEventType, +) + + +def _reverse_readline(path: Path, buf_size: int = 8192) -> Iterator[str]: + """Read a file line by line from the end without loading entire file into memory. + + Yields lines in reverse order (last line first). + """ + with path.open('rb') as f: + f.seek(0, 2) # Seek to end + file_size = f.tell() + if file_size == 0: + return + + remaining = file_size + buffer = b'' + + while remaining > 0: + # Read a chunk from the end + read_size = min(buf_size, remaining) + remaining -= read_size + f.seek(remaining) + chunk = f.read(read_size) + buffer = chunk + buffer + + # Yield complete lines from buffer + while b'\n' in buffer: + # Find the last newline + newline_pos = buffer.rfind(b'\n') + if newline_pos == len(buffer) - 1: + # Trailing newline, look for previous one + newline_pos = buffer.rfind(b'\n', 0, newline_pos) + if newline_pos == -1: + break + # Yield the line after this newline + line = buffer[newline_pos + 1 :] + buffer = buffer[: newline_pos + 1] + if line.strip(): + yield line.decode('utf-8', errors='replace') + + # Yield any remaining content as the first line of the file + if buffer.strip(): + yield buffer.decode('utf-8', errors='replace') + + +def _extract_model(entry: dict) -> Optional[str]: + """Extract model from a transcript entry (top level or nested in message).""" + return entry.get('model') or (entry.get('message') or {}).get('model') + + +def _extract_generation_id(entry: dict) -> Optional[str]: + """Extract generation ID from a user-type transcript entry.""" + if entry.get('type') == 'user': + return entry.get('uuid') + return None + + +def _extract_from_claude_transcript( + transcript_path: str, +) -> tuple[Optional[str], Optional[str], Optional[str]]: + """Extract IDE version, model, and latest generation ID from Claude Code transcript file. + + The transcript is a JSONL file where each line is a JSON object. + We look for 'version' (IDE version), 'model', and 'uuid' (generation ID) fields. + The generation_id is the UUID of the latest 'user' type message. + + Scans from end to start since latest entries are at the end. + Uses reverse reading to avoid loading entire file into memory. + + Returns: + Tuple of (ide_version, model, generation_id), any may be None if not found. + """ + if not transcript_path: + return None, None, None + + path = Path(transcript_path) + if not path.exists(): + return None, None, None + + ide_version = None + model = None + generation_id = None + + try: + for line in _reverse_readline(path): + line = line.strip() + if not line: + continue + try: + entry = json.loads(line) + ide_version = ide_version or entry.get('version') + model = model or _extract_model(entry) + generation_id = generation_id or _extract_generation_id(entry) + + if ide_version and model and generation_id: + break + except json.JSONDecodeError: + continue + except OSError: + pass + + return ide_version, model, generation_id + + +@dataclass +class AIHookPayload: + """Unified payload object that normalizes field names from different AI tools.""" + + # Event identification + event_name: str # Canonical event type (e.g., 'prompt', 'file_read', 'mcp_execution') + conversation_id: Optional[str] = None + generation_id: Optional[str] = None + + # User and IDE information + ide_user_email: Optional[str] = None + model: Optional[str] = None + ide_provider: str = None # AIIDEType value (e.g., 'cursor', 'claude-code') + ide_version: Optional[str] = None + + # Event-specific data + prompt: Optional[str] = None # For prompt events + file_path: Optional[str] = None # For file_read events + mcp_server_name: Optional[str] = None # For mcp_execution events + mcp_tool_name: Optional[str] = None # For mcp_execution events + mcp_arguments: Optional[dict] = None # For mcp_execution events + + @classmethod + def from_cursor_payload(cls, payload: dict) -> 'AIHookPayload': + """Create AIHookPayload from Cursor IDE payload. + + Maps Cursor-specific event names to canonical event types. + """ + cursor_event_name = payload.get('hook_event_name', '') + # Map Cursor event name to canonical type, fallback to original if not found + canonical_event = CURSOR_EVENT_MAPPING.get(cursor_event_name, cursor_event_name) + + return cls( + event_name=canonical_event, + conversation_id=payload.get('conversation_id'), + generation_id=payload.get('generation_id'), + ide_user_email=payload.get('user_email'), + model=payload.get('model'), + ide_provider=AIIDEType.CURSOR.value, + ide_version=payload.get('cursor_version'), + prompt=payload.get('prompt', ''), + file_path=payload.get('file_path') or payload.get('path'), + mcp_server_name=payload.get('command'), # MCP server name + mcp_tool_name=payload.get('tool_name') or payload.get('tool'), + mcp_arguments=payload.get('arguments') or payload.get('tool_input') or payload.get('input'), + ) + + @classmethod + def from_claude_code_payload(cls, payload: dict) -> 'AIHookPayload': + """Create AIHookPayload from Claude Code IDE payload. + + Claude Code has a different structure: + - hook_event_name: 'UserPromptSubmit' or 'PreToolUse' + - For PreToolUse: tool_name determines if it's file read ('Read') or MCP ('mcp__*') + - tool_input contains tool arguments (e.g., file_path for Read tool) + - transcript_path points to JSONL file with version and model info + """ + hook_event_name = payload.get('hook_event_name', '') + tool_name = payload.get('tool_name', '') + tool_input = payload.get('tool_input') + + if hook_event_name == 'UserPromptSubmit': + canonical_event = AiHookEventType.PROMPT + elif hook_event_name == 'PreToolUse': + canonical_event = AiHookEventType.FILE_READ if tool_name == 'Read' else AiHookEventType.MCP_EXECUTION + else: + # Unknown event, use the raw event name + canonical_event = CLAUDE_CODE_EVENT_MAPPING.get(hook_event_name, hook_event_name) + + # Extract file_path from tool_input for Read tool + file_path = None + if tool_name == 'Read' and isinstance(tool_input, dict): + file_path = tool_input.get('file_path') + + # For MCP tools, the entire tool_input is the arguments + mcp_arguments = tool_input if tool_name.startswith('mcp__') else None + + # Extract MCP server and tool name from tool_name (format: mcp____) + mcp_server_name = None + mcp_tool_name = None + if tool_name.startswith('mcp__'): + parts = tool_name.split('__') + if len(parts) >= 2: + mcp_server_name = parts[1] + if len(parts) >= 3: + mcp_tool_name = parts[2] + + # Extract IDE version, model, and generation ID from transcript file + ide_version, model, generation_id = _extract_from_claude_transcript(payload.get('transcript_path')) + + return cls( + event_name=canonical_event, + conversation_id=payload.get('session_id'), + generation_id=generation_id, + ide_user_email=None, # Claude Code doesn't provide this in hook payload + model=model, + ide_provider=AIIDEType.CLAUDE_CODE.value, + ide_version=ide_version, + prompt=payload.get('prompt', ''), + file_path=file_path, + mcp_server_name=mcp_server_name, + mcp_tool_name=mcp_tool_name, + mcp_arguments=mcp_arguments, + ) + + @staticmethod + def is_payload_for_ide(payload: dict, ide: str) -> bool: + """Check if the payload's event name matches the expected IDE. + + This prevents double-processing when Cursor reads Claude Code hooks + or vice versa. If the payload's hook_event_name doesn't match the + expected IDE's event names, we should skip processing. + + Args: + payload: The raw payload from the IDE + ide: The IDE name or AIIDEType enum value + + Returns: + True if the payload matches the IDE, False otherwise. + """ + hook_event_name = payload.get('hook_event_name', '') + + if ide == AIIDEType.CLAUDE_CODE: + return hook_event_name in CLAUDE_CODE_EVENT_NAMES + if ide == AIIDEType.CURSOR: + return hook_event_name in CURSOR_EVENT_NAMES + + # Unknown IDE, allow processing + return True + + @classmethod + def from_payload(cls, payload: dict, tool: str = AIIDEType.CURSOR.value) -> 'AIHookPayload': + """Create AIHookPayload from any tool's payload. + + Args: + payload: The raw payload from the IDE + tool: The IDE/tool name or AIIDEType enum value + + Returns: + AIHookPayload instance + + Raises: + ValueError: If the tool is not supported + """ + if tool == AIIDEType.CURSOR: + return cls.from_cursor_payload(payload) + if tool == AIIDEType.CLAUDE_CODE: + return cls.from_claude_code_payload(payload) + raise ValueError(f'Unsupported IDE/tool: {tool}') diff --git a/cycode/cli/apps/ai_guardrails/scan/policy.py b/cycode/cli/apps/ai_guardrails/scan/policy.py new file mode 100644 index 00000000..f40d77c0 --- /dev/null +++ b/cycode/cli/apps/ai_guardrails/scan/policy.py @@ -0,0 +1,85 @@ +""" +Policy loading and configuration management for AI guardrails. + +Policies are loaded and merged in order (later overrides earlier): +1. Built-in defaults (consts.DEFAULT_POLICY) +2. User-level config (~/.cycode/ai-guardrails.yaml) +3. Repo-level config (/.cycode/ai-guardrails.yaml) +""" + +import json +from pathlib import Path +from typing import Any, Optional + +import yaml + +from cycode.cli.apps.ai_guardrails.scan.consts import DEFAULT_POLICY, POLICY_FILE_NAME + + +def deep_merge(base: dict, override: dict) -> dict: + """Deep merge two dictionaries, with override taking precedence.""" + result = base.copy() + for key, value in override.items(): + if key in result and isinstance(result[key], dict) and isinstance(value, dict): + result[key] = deep_merge(result[key], value) + else: + result[key] = value + return result + + +def load_yaml_file(path: Path) -> Optional[dict]: + """Load a YAML or JSON config file.""" + if not path.exists(): + return None + try: + content = path.read_text(encoding='utf-8') + if path.suffix in ('.yaml', '.yml'): + return yaml.safe_load(content) + return json.loads(content) + except Exception: + return None + + +def load_defaults() -> dict: + """Load built-in defaults.""" + return DEFAULT_POLICY.copy() + + +def get_policy_value(policy: dict, *keys: str, default: Any = None) -> Any: + """Get a nested value from the policy dict.""" + current = policy + for key in keys: + if not isinstance(current, dict): + return default + current = current.get(key) + if current is None: + return default + return current + + +def load_policy(workspace_root: Optional[str] = None) -> dict: + """ + Load policy by merging configs in order of precedence. + + Merge order: defaults <- user config <- repo config + + Args: + workspace_root: Workspace root path for repo-level config lookup. + """ + # Start with defaults + policy = load_defaults() + + # Merge user-level config (if exists) + user_policy_path = Path.home() / '.cycode' / POLICY_FILE_NAME + user_config = load_yaml_file(user_policy_path) + if user_config: + policy = deep_merge(policy, user_config) + + # Merge repo-level config (if exists) - highest precedence + if workspace_root: + repo_policy_path = Path(workspace_root) / '.cycode' / POLICY_FILE_NAME + repo_config = load_yaml_file(repo_policy_path) + if repo_config: + policy = deep_merge(policy, repo_config) + + return policy diff --git a/cycode/cli/apps/ai_guardrails/scan/response_builders.py b/cycode/cli/apps/ai_guardrails/scan/response_builders.py new file mode 100644 index 00000000..ff0a6aa4 --- /dev/null +++ b/cycode/cli/apps/ai_guardrails/scan/response_builders.py @@ -0,0 +1,135 @@ +""" +Response builders for different AI IDE hooks. + +Each IDE has its own response format for hooks. This module provides +an abstract interface and concrete implementations for each supported IDE. +""" + +from abc import ABC, abstractmethod + +from cycode.cli.apps.ai_guardrails.consts import AIIDEType + + +class IDEResponseBuilder(ABC): + """Abstract base class for IDE-specific response builders.""" + + @abstractmethod + def allow_permission(self) -> dict: + """Build response to allow file read or MCP execution.""" + + @abstractmethod + def deny_permission(self, user_message: str, agent_message: str) -> dict: + """Build response to deny file read or MCP execution.""" + + @abstractmethod + def ask_permission(self, user_message: str, agent_message: str) -> dict: + """Build response to ask user for permission (warn mode).""" + + @abstractmethod + def allow_prompt(self) -> dict: + """Build response to allow prompt submission.""" + + @abstractmethod + def deny_prompt(self, user_message: str) -> dict: + """Build response to deny prompt submission.""" + + +class CursorResponseBuilder(IDEResponseBuilder): + """Response builder for Cursor IDE hooks. + + Cursor hook response formats: + - beforeSubmitPrompt: {"continue": bool, "user_message": str} + - beforeReadFile: {"permission": str, "user_message": str, "agent_message": str} + - beforeMCPExecution: {"permission": str, "user_message": str, "agent_message": str} + """ + + def allow_permission(self) -> dict: + """Allow file read or MCP execution.""" + return {'permission': 'allow'} + + def deny_permission(self, user_message: str, agent_message: str) -> dict: + """Deny file read or MCP execution.""" + return {'permission': 'deny', 'user_message': user_message, 'agent_message': agent_message} + + def ask_permission(self, user_message: str, agent_message: str) -> dict: + """Ask user for permission (warn mode).""" + return {'permission': 'ask', 'user_message': user_message, 'agent_message': agent_message} + + def allow_prompt(self) -> dict: + """Allow prompt submission.""" + return {'continue': True} + + def deny_prompt(self, user_message: str) -> dict: + """Deny prompt submission.""" + return {'continue': False, 'user_message': user_message} + + +class ClaudeCodeResponseBuilder(IDEResponseBuilder): + """Response builder for Claude Code IDE hooks. + + Claude Code hook response formats: + - UserPromptSubmit: {} for allow, {"decision": "block", "reason": str} for deny + - PreToolUse: hookSpecificOutput with permissionDecision (allow/deny/ask) + """ + + def allow_permission(self) -> dict: + """Allow file read or MCP execution.""" + return { + 'hookSpecificOutput': { + 'hookEventName': 'PreToolUse', + 'permissionDecision': 'allow', + } + } + + def deny_permission(self, user_message: str, agent_message: str) -> dict: + """Deny file read or MCP execution.""" + return { + 'hookSpecificOutput': { + 'hookEventName': 'PreToolUse', + 'permissionDecision': 'deny', + 'permissionDecisionReason': user_message, + } + } + + def ask_permission(self, user_message: str, agent_message: str) -> dict: + """Ask user for permission (warn mode).""" + return { + 'hookSpecificOutput': { + 'hookEventName': 'PreToolUse', + 'permissionDecision': 'ask', + 'permissionDecisionReason': user_message, + } + } + + def allow_prompt(self) -> dict: + """Allow prompt submission (empty response means allow).""" + return {} + + def deny_prompt(self, user_message: str) -> dict: + """Deny prompt submission.""" + return {'decision': 'block', 'reason': user_message} + + +# Registry of response builders by IDE type +_RESPONSE_BUILDERS: dict[str, IDEResponseBuilder] = { + AIIDEType.CURSOR: CursorResponseBuilder(), + AIIDEType.CLAUDE_CODE: ClaudeCodeResponseBuilder(), +} + + +def get_response_builder(ide: str = AIIDEType.CURSOR.value) -> IDEResponseBuilder: + """Get the response builder for a specific IDE. + + Args: + ide: The IDE name (e.g., 'cursor', 'claude-code') or AIIDEType enum + + Returns: + IDEResponseBuilder instance for the specified IDE + + Raises: + ValueError: If the IDE is not supported + """ + builder = _RESPONSE_BUILDERS.get(ide.lower()) + if not builder: + raise ValueError(f'Unsupported IDE: {ide}. Supported IDEs: {list(_RESPONSE_BUILDERS.keys())}') + return builder diff --git a/cycode/cli/apps/ai_guardrails/scan/scan_command.py b/cycode/cli/apps/ai_guardrails/scan/scan_command.py new file mode 100644 index 00000000..add2bb83 --- /dev/null +++ b/cycode/cli/apps/ai_guardrails/scan/scan_command.py @@ -0,0 +1,142 @@ +""" +Scan command for AI guardrails. + +This command handles AI IDE hooks by reading JSON from stdin and outputting +a JSON response to stdout. It scans prompts, file reads, and MCP tool calls +for secrets before they are sent to AI models. + +Supports multiple IDEs with different hook event types. The specific hook events +supported depend on the IDE being used (e.g., Cursor supports beforeSubmitPrompt, +beforeReadFile, beforeMCPExecution). +""" + +import sys +from typing import Annotated + +import click +import typer + +from cycode.cli.apps.ai_guardrails.consts import AIIDEType +from cycode.cli.apps.ai_guardrails.scan.handlers import get_handler_for_event +from cycode.cli.apps.ai_guardrails.scan.payload import AIHookPayload +from cycode.cli.apps.ai_guardrails.scan.policy import load_policy +from cycode.cli.apps.ai_guardrails.scan.response_builders import get_response_builder +from cycode.cli.apps.ai_guardrails.scan.types import AiHookEventType +from cycode.cli.apps.ai_guardrails.scan.utils import output_json, safe_json_parse +from cycode.cli.exceptions.custom_exceptions import HttpUnauthorizedError +from cycode.cli.utils.get_api_client import get_ai_security_manager_client, get_scan_cycode_client +from cycode.logger import get_logger + +logger = get_logger('AI Guardrails') + + +def _get_auth_error_message(error: Exception) -> str: + """Get user-friendly message for authentication errors.""" + if isinstance(error, click.ClickException): + # Missing credentials + return f'{error.message} Please run `cycode auth` to set up your credentials.' + + if isinstance(error, HttpUnauthorizedError): + # Invalid/expired credentials + return ( + 'Unable to authenticate to Cycode. Your credentials are invalid or have expired. ' + 'Please run `cycode auth` to update your credentials.' + ) + + # Fallback + return 'Authentication failed. Please run `cycode auth` to set up your credentials.' + + +def _initialize_clients(ctx: typer.Context) -> None: + """Initialize API clients. + + May raise click.ClickException if credentials are missing, + or HttpUnauthorizedError if credentials are invalid. + """ + scan_client = get_scan_cycode_client(ctx) + ctx.obj['client'] = scan_client + + ai_security_client = get_ai_security_manager_client(ctx) + ctx.obj['ai_security_client'] = ai_security_client + + +def scan_command( + ctx: typer.Context, + ide: Annotated[ + str, + typer.Option( + '--ide', + help='IDE that sent the payload (e.g., "cursor"). Defaults to cursor.', + hidden=True, + ), + ] = AIIDEType.CURSOR.value, +) -> None: + """Scan content from AI IDE hooks for secrets. + + This command reads a JSON payload from stdin containing hook event data + and outputs a JSON response to stdout indicating whether to allow or block the action. + + The hook event type is determined from the event field in the payload (field name + varies by IDE). Each IDE may support different hook events for scanning prompts, + file access, and tool executions. + + Example usage (from IDE hooks configuration): + { "command": "cycode ai-guardrails scan" } + """ + stdin_data = sys.stdin.read().strip() + payload = safe_json_parse(stdin_data) + + tool = ide.lower() + response_builder = get_response_builder(tool) + + if not payload: + logger.debug('Empty or invalid JSON payload received') + output_json(response_builder.allow_prompt()) + return + + # Check if the payload matches the expected IDE - prevents double-processing + # when Cursor reads Claude Code hooks from ~/.claude/settings.json + if not AIHookPayload.is_payload_for_ide(payload, tool): + logger.debug( + 'Payload event does not match expected IDE, skipping', + extra={'hook_event_name': payload.get('hook_event_name'), 'expected_ide': tool}, + ) + output_json(response_builder.allow_prompt()) + return + + unified_payload = AIHookPayload.from_payload(payload, tool=tool) + event_name = unified_payload.event_name + logger.debug('Processing AI guardrails hook', extra={'event_name': event_name, 'tool': tool}) + + workspace_roots = payload.get('workspace_roots', ['.']) + policy = load_policy(workspace_roots[0]) + + try: + _initialize_clients(ctx) + + handler = get_handler_for_event(event_name) + if handler is None: + logger.debug('Unknown hook event, allowing by default', extra={'event_name': event_name}) + output_json(response_builder.allow_prompt()) + return + + response = handler(ctx, unified_payload, policy) + logger.debug('Hook handler completed', extra={'event_name': event_name, 'response': response}) + output_json(response) + + except (click.ClickException, HttpUnauthorizedError) as e: + error_message = _get_auth_error_message(e) + if event_name == AiHookEventType.PROMPT: + output_json(response_builder.deny_prompt(error_message)) + return + output_json(response_builder.deny_permission(error_message, 'Authentication required')) + + except Exception as e: + logger.error('Hook handler failed', exc_info=e) + if policy.get('fail_open', True): + output_json(response_builder.allow_prompt()) + return + if event_name == AiHookEventType.PROMPT: + output_json(response_builder.deny_prompt('Cycode guardrails error - blocking due to fail-closed policy')) + return + output_json(response_builder.deny_permission('Cycode guardrails error', 'Blocking due to fail-closed policy')) diff --git a/cycode/cli/apps/ai_guardrails/scan/types.py b/cycode/cli/apps/ai_guardrails/scan/types.py new file mode 100644 index 00000000..585c7820 --- /dev/null +++ b/cycode/cli/apps/ai_guardrails/scan/types.py @@ -0,0 +1,65 @@ +"""Type definitions for AI guardrails.""" + +import sys + +if sys.version_info >= (3, 11): + from enum import StrEnum +else: + from enum import Enum + + class StrEnum(str, Enum): + def __str__(self) -> str: + return self.value + + +class AiHookEventType(StrEnum): + """Canonical event types for AI guardrails. + + These are IDE-agnostic event types. Each IDE's specific event names + are mapped to these canonical types using the mapping dictionaries below. + """ + + PROMPT = 'Prompt' + FILE_READ = 'FileRead' + MCP_EXECUTION = 'McpExecution' + + +# IDE-specific event name mappings to canonical types +CURSOR_EVENT_MAPPING = { + 'beforeSubmitPrompt': AiHookEventType.PROMPT, + 'beforeReadFile': AiHookEventType.FILE_READ, + 'beforeMCPExecution': AiHookEventType.MCP_EXECUTION, +} + +# Claude Code event mapping - note that PreToolUse requires tool_name inspection +# to determine the actual event type (file read vs MCP execution) +CLAUDE_CODE_EVENT_MAPPING = { + 'UserPromptSubmit': AiHookEventType.PROMPT, + 'PreToolUse': None, # Requires tool_name inspection to determine actual type +} + +# Set of known event names per IDE (for IDE detection) +CURSOR_EVENT_NAMES = set(CURSOR_EVENT_MAPPING.keys()) +CLAUDE_CODE_EVENT_NAMES = set(CLAUDE_CODE_EVENT_MAPPING.keys()) + + +class AIHookOutcome(StrEnum): + """Outcome of an AI hook event evaluation.""" + + ALLOWED = 'allowed' + BLOCKED = 'blocked' + WARNED = 'warned' + + +class BlockReason(StrEnum): + """Reason why an AI hook event was blocked. + + These are categorical reasons sent to the backend for tracking/analytics, + separate from the detailed user-facing messages. + """ + + SECRETS_IN_PROMPT = 'secrets_in_prompt' + SECRETS_IN_FILE = 'secrets_in_file' + SECRETS_IN_MCP_ARGS = 'secrets_in_mcp_args' + SENSITIVE_PATH = 'sensitive_path' + SCAN_FAILURE = 'scan_failure' diff --git a/cycode/cli/apps/ai_guardrails/scan/utils.py b/cycode/cli/apps/ai_guardrails/scan/utils.py new file mode 100644 index 00000000..e14c1c02 --- /dev/null +++ b/cycode/cli/apps/ai_guardrails/scan/utils.py @@ -0,0 +1,72 @@ +""" +Utility functions for AI guardrails. + +Includes JSON parsing, path matching, and text handling utilities. +""" + +import json +import os +from pathlib import Path + +from cycode.cli.apps.ai_guardrails.scan.policy import get_policy_value + + +def safe_json_parse(s: str) -> dict: + """Parse JSON string, returning empty dict on failure.""" + try: + return json.loads(s) if s else {} + except (json.JSONDecodeError, TypeError): + return {} + + +def truncate_utf8(text: str, max_bytes: int) -> str: + """Truncate text to max bytes while preserving valid UTF-8.""" + if not text: + return '' + encoded = text.encode('utf-8') + if len(encoded) <= max_bytes: + return text + return encoded[:max_bytes].decode('utf-8', errors='ignore') + + +def normalize_path(file_path: str) -> str: + """Normalize path to prevent traversal attacks.""" + if not file_path: + return '' + normalized = os.path.normpath(file_path) + # Reject paths that attempt to escape outside bounds + if normalized.startswith('..'): + return '' + return normalized + + +def matches_glob(file_path: str, pattern: str) -> bool: + """Check if file path matches a glob pattern. + + Case-insensitive matching for cross-platform compatibility. + """ + normalized = normalize_path(file_path) + if not normalized or not pattern: + return False + + path = Path(normalized) + # Try case-sensitive first + if path.match(pattern): + return True + + # Then try case-insensitive by lowercasing both path and pattern + path_lower = Path(normalized.lower()) + return path_lower.match(pattern.lower()) + + +def is_denied_path(file_path: str, policy: dict) -> bool: + """Check if file path is in the denylist.""" + if not file_path: + return False + globs = get_policy_value(policy, 'file_read', 'deny_globs', default=[]) + return any(matches_glob(file_path, g) for g in globs) + + +def output_json(obj: dict) -> None: + """Write JSON response to stdout (for IDE to read).""" + print(json.dumps(obj), end='') # noqa: T201 diff --git a/cycode/cli/apps/ai_guardrails/status_command.py b/cycode/cli/apps/ai_guardrails/status_command.py new file mode 100644 index 00000000..ee1e5bcf --- /dev/null +++ b/cycode/cli/apps/ai_guardrails/status_command.py @@ -0,0 +1,98 @@ +"""Status command for AI guardrails hooks.""" + +import os +from pathlib import Path +from typing import Annotated, Optional + +import typer +from rich.table import Table + +from cycode.cli.apps.ai_guardrails.command_utils import console, validate_and_parse_ide, validate_scope +from cycode.cli.apps.ai_guardrails.consts import IDE_CONFIGS, AIIDEType +from cycode.cli.apps.ai_guardrails.hooks_manager import get_hooks_status + + +def status_command( + ctx: typer.Context, + scope: Annotated[ + str, + typer.Option( + '--scope', + '-s', + help='Check scope: "user", "repo", or "all" for both.', + ), + ] = 'all', + ide: Annotated[ + str, + typer.Option( + '--ide', + help='IDE to check status for (e.g., "cursor", "claude-code", or "all" for all IDEs). Defaults to cursor.', + ), + ] = AIIDEType.CURSOR.value, + repo_path: Annotated[ + Optional[Path], + typer.Option( + '--repo-path', + help='Repository path for repo-scoped status (defaults to current directory).', + exists=True, + file_okay=False, + dir_okay=True, + resolve_path=True, + ), + ] = None, +) -> None: + """Show AI guardrails hook installation status. + + Displays the current status of Cycode AI guardrails hooks for the specified IDE. + + Examples: + cycode ai-guardrails status # Show both user and repo status + cycode ai-guardrails status --scope user # Show only user-level status + cycode ai-guardrails status --scope repo # Show only repo-level status + cycode ai-guardrails status --ide cursor # Check status for Cursor IDE + cycode ai-guardrails status --ide all # Check status for all supported IDEs + """ + # Validate inputs (status allows 'all' scope) + validate_scope(scope, allowed_scopes=('user', 'repo', 'all')) + if repo_path is None: + repo_path = Path(os.getcwd()) + ide_type = validate_and_parse_ide(ide) + + ides_to_check: list[AIIDEType] = list(AIIDEType) if ide_type is None else [ide_type] + + scopes_to_check = ['user', 'repo'] if scope == 'all' else [scope] + + for current_ide in ides_to_check: + ide_name = IDE_CONFIGS[current_ide].name + console.print() + console.print(f'[bold cyan]═══ {ide_name} ═══[/]') + + for check_scope in scopes_to_check: + status = get_hooks_status(check_scope, repo_path if check_scope == 'repo' else None, ide=current_ide) + + console.print() + console.print(f'[bold]{check_scope.upper()} SCOPE[/]') + console.print(f'Path: {status["hooks_path"]}') + + if not status['file_exists']: + console.print('[dim]No hooks file found[/]') + continue + + if status['cycode_installed']: + console.print('[green]✓ Cycode AI guardrails: INSTALLED[/]') + else: + console.print('[yellow]○ Cycode AI guardrails: NOT INSTALLED[/]') + + # Show hook details + table = Table(show_header=True, header_style='bold') + table.add_column('Hook Event') + table.add_column('Cycode Enabled') + table.add_column('Total Hooks') + + for event, info in status['hooks'].items(): + enabled = '[green]Yes[/]' if info['enabled'] else '[dim]No[/]' + table.add_row(event, enabled, str(info['total_entries'])) + + console.print(table) + + console.print() diff --git a/cycode/cli/apps/ai_guardrails/uninstall_command.py b/cycode/cli/apps/ai_guardrails/uninstall_command.py new file mode 100644 index 00000000..f7b8341c --- /dev/null +++ b/cycode/cli/apps/ai_guardrails/uninstall_command.py @@ -0,0 +1,89 @@ +"""Uninstall command for AI guardrails hooks.""" + +from pathlib import Path +from typing import Annotated, Optional + +import typer + +from cycode.cli.apps.ai_guardrails.command_utils import ( + console, + resolve_repo_path, + validate_and_parse_ide, + validate_scope, +) +from cycode.cli.apps.ai_guardrails.consts import IDE_CONFIGS, AIIDEType +from cycode.cli.apps.ai_guardrails.hooks_manager import uninstall_hooks + + +def uninstall_command( + ctx: typer.Context, + scope: Annotated[ + str, + typer.Option( + '--scope', + '-s', + help='Uninstall scope: "user" for user-level hooks, "repo" for repository-level hooks.', + ), + ] = 'user', + ide: Annotated[ + str, + typer.Option( + '--ide', + help='IDE to uninstall hooks from (e.g., "cursor", "claude-code", "all"). Defaults to cursor.', + ), + ] = AIIDEType.CURSOR.value, + repo_path: Annotated[ + Optional[Path], + typer.Option( + '--repo-path', + help='Repository path for repo-scoped uninstallation (defaults to current directory).', + exists=True, + file_okay=False, + dir_okay=True, + resolve_path=True, + ), + ] = None, +) -> None: + """Remove AI guardrails hooks from supported IDEs. + + This command removes Cycode hooks from the IDE's hooks configuration. + Other hooks (if any) will be preserved. + + Examples: + cycode ai-guardrails uninstall # Remove user-level hooks + cycode ai-guardrails uninstall --scope repo # Remove repo-level hooks + cycode ai-guardrails uninstall --ide cursor # Uninstall from Cursor IDE + cycode ai-guardrails uninstall --ide all # Uninstall from all supported IDEs + """ + # Validate inputs + validate_scope(scope) + repo_path = resolve_repo_path(scope, repo_path) + ide_type = validate_and_parse_ide(ide) + + ides_to_uninstall: list[AIIDEType] = list(AIIDEType) if ide_type is None else [ide_type] + + results: list[tuple[str, bool, str]] = [] + for current_ide in ides_to_uninstall: + ide_name = IDE_CONFIGS[current_ide].name + success, message = uninstall_hooks(scope, repo_path, ide=current_ide) + results.append((ide_name, success, message)) + + # Report results for each IDE + any_success = False + all_success = True + for _ide_name, success, message in results: + if success: + console.print(f'[green]✓[/] {message}') + any_success = True + else: + console.print(f'[red]✗[/] {message}', style='bold red') + all_success = False + + if any_success: + console.print() + successful_ides = [name for name, success, _ in results if success] + ide_list = ', '.join(successful_ides) + console.print(f'[dim]Restart {ide_list} for changes to take effect.[/]') + + if not all_success: + raise typer.Exit(1) diff --git a/cycode/cli/apps/ai_remediation/__init__.py b/cycode/cli/apps/ai_remediation/__init__.py new file mode 100644 index 00000000..00d0c7c5 --- /dev/null +++ b/cycode/cli/apps/ai_remediation/__init__.py @@ -0,0 +1,20 @@ +import typer + +from cycode.cli.apps.ai_remediation.ai_remediation_command import ai_remediation_command + +app = typer.Typer() + +_ai_remediation_epilog = ( + 'Note: AI remediation suggestions are generated automatically and should be reviewed before applying.' +) + +app.command( + name='ai-remediation', + short_help='Get AI remediation (INTERNAL).', + epilog=_ai_remediation_epilog, + hidden=True, + no_args_is_help=True, +)(ai_remediation_command) + +# backward compatibility +app.command(hidden=True, name='ai_remediation')(ai_remediation_command) diff --git a/cycode/cli/apps/ai_remediation/ai_remediation_command.py b/cycode/cli/apps/ai_remediation/ai_remediation_command.py new file mode 100644 index 00000000..ab2eca5e --- /dev/null +++ b/cycode/cli/apps/ai_remediation/ai_remediation_command.py @@ -0,0 +1,39 @@ +from typing import Annotated +from uuid import UUID + +import typer + +from cycode.cli.apps.ai_remediation.apply_fix import apply_fix +from cycode.cli.apps.ai_remediation.print_remediation import print_remediation +from cycode.cli.exceptions.handle_ai_remediation_errors import handle_ai_remediation_exception +from cycode.cli.utils.get_api_client import get_scan_cycode_client + + +def ai_remediation_command( + ctx: typer.Context, + detection_id: Annotated[UUID, typer.Argument(help='Detection ID to get remediation for', show_default=False)], + fix: Annotated[ + bool, typer.Option('--fix', help='Apply fixes to resolve violations. Note: fix could be not available.') + ] = False, +) -> None: + """:robot: [bold cyan]Get AI-powered remediation for security issues.[/] + + This command provides AI-generated remediation guidance for detected security issues. + + Example usage: + * `cycode ai-remediation `: View remediation guidance + * `cycode ai-remediation --fix`: Apply suggested fixes + """ + client = get_scan_cycode_client(ctx) + + try: + remediation_markdown = client.get_ai_remediation(detection_id) + fix_diff = client.get_ai_remediation(detection_id, fix=True) + is_fix_available = bool(fix_diff) # exclude empty string, None, etc. + + if fix: + apply_fix(ctx, fix_diff, is_fix_available) + else: + print_remediation(ctx, remediation_markdown, is_fix_available) + except Exception as err: + handle_ai_remediation_exception(ctx, err) diff --git a/cycode/cli/apps/ai_remediation/apply_fix.py b/cycode/cli/apps/ai_remediation/apply_fix.py new file mode 100644 index 00000000..bd840411 --- /dev/null +++ b/cycode/cli/apps/ai_remediation/apply_fix.py @@ -0,0 +1,24 @@ +import os + +import typer +from patch_ng import fromstring + +from cycode.cli.models import CliResult + + +def apply_fix(ctx: typer.Context, diff: str, is_fix_available: bool) -> None: + printer = ctx.obj.get('console_printer') + if not is_fix_available: + printer.print_result(CliResult(success=False, message='Fix is not available for this violation')) + return + + patch = fromstring(diff.encode('UTF-8')) + if patch is False: + printer.print_result(CliResult(success=False, message='Failed to parse fix diff')) + return + + is_fix_applied = patch.apply(root=os.getcwd(), strip=0) + if is_fix_applied: + printer.print_result(CliResult(success=True, message='Fix applied successfully')) + else: + printer.print_result(CliResult(success=False, message='Failed to apply fix')) diff --git a/cycode/cli/apps/ai_remediation/print_remediation.py b/cycode/cli/apps/ai_remediation/print_remediation.py new file mode 100644 index 00000000..92272b76 --- /dev/null +++ b/cycode/cli/apps/ai_remediation/print_remediation.py @@ -0,0 +1,14 @@ +import typer +from rich.markdown import Markdown + +from cycode.cli.console import console +from cycode.cli.models import CliResult + + +def print_remediation(ctx: typer.Context, remediation_markdown: str, is_fix_available: bool) -> None: + printer = ctx.obj.get('console_printer') + if printer.is_json_printer: + data = {'remediation': remediation_markdown, 'is_fix_available': is_fix_available} + printer.print_result(CliResult(success=True, message='Remediation fetched successfully', data=data)) + else: # text or table + console.print(Markdown(remediation_markdown)) diff --git a/cycode/cli/apps/auth/__init__.py b/cycode/cli/apps/auth/__init__.py new file mode 100644 index 00000000..f487e1bf --- /dev/null +++ b/cycode/cli/apps/auth/__init__.py @@ -0,0 +1,9 @@ +import typer + +from cycode.cli.apps.auth.auth_command import auth_command + +_auth_command_docs = 'https://github.com/cycodehq/cycode-cli/blob/main/README.md#using-the-auth-command' +_auth_command_epilog = f'[bold]Documentation:[/] [link={_auth_command_docs}]{_auth_command_docs}[/link]' + +app = typer.Typer(no_args_is_help=False) +app.command(name='auth', epilog=_auth_command_epilog, short_help='Authenticate your machine with Cycode.')(auth_command) diff --git a/cycode/cli/apps/auth/auth_command.py b/cycode/cli/apps/auth/auth_command.py new file mode 100644 index 00000000..1184a916 --- /dev/null +++ b/cycode/cli/apps/auth/auth_command.py @@ -0,0 +1,29 @@ +import typer + +from cycode.cli.apps.auth.auth_manager import AuthManager +from cycode.cli.exceptions.handle_auth_errors import handle_auth_exception +from cycode.cli.logger import logger +from cycode.cli.models import CliResult + + +def auth_command(ctx: typer.Context) -> None: + """:key: [bold cyan]Authenticate your machine with Cycode.[/] + + This command handles authentication with Cycode's security platform. + + Example usage: + * `cycode auth`: Start interactive authentication + * `cycode auth --help`: View authentication options + """ + printer = ctx.obj.get('console_printer') + + try: + logger.debug('Starting authentication process') + + auth_manager = AuthManager() + auth_manager.authenticate() + + result = CliResult(success=True, message='Successfully logged into cycode') + printer.print_result(result) + except Exception as err: + handle_auth_exception(ctx, err) diff --git a/cycode/cli/apps/auth/auth_common.py b/cycode/cli/apps/auth/auth_common.py new file mode 100644 index 00000000..f4ea09d9 --- /dev/null +++ b/cycode/cli/apps/auth/auth_common.py @@ -0,0 +1,69 @@ +from typing import TYPE_CHECKING, Optional + +from cycode.cli.apps.auth.models import AuthInfo +from cycode.cli.exceptions.custom_exceptions import HttpUnauthorizedError, RequestHttpError +from cycode.cli.user_settings.credentials_manager import CredentialsManager +from cycode.cli.utils.jwt_utils import get_user_and_tenant_ids_from_access_token +from cycode.cyclient.cycode_oidc_based_client import CycodeOidcBasedClient +from cycode.cyclient.cycode_token_based_client import CycodeTokenBasedClient + +if TYPE_CHECKING: + from typer import Context + + +def get_authorization_info(ctx: 'Context') -> Optional[AuthInfo]: + printer = ctx.obj.get('console_printer') + + client_id = ctx.obj.get('client_id') + client_secret = ctx.obj.get('client_secret') + id_token = ctx.obj.get('id_token') + + credentials_manager = CredentialsManager() + + auth_info = _try_oidc_authorization(ctx, printer, client_id, id_token) + if auth_info: + return auth_info + + if not client_id or not client_secret: + stored_client_id, stored_id_token = credentials_manager.get_oidc_credentials() + auth_info = _try_oidc_authorization(ctx, printer, stored_client_id, stored_id_token) + if auth_info: + return auth_info + + client_id, client_secret = credentials_manager.get_credentials() + + if not client_id or not client_secret: + return None + + try: + access_token = CycodeTokenBasedClient(client_id, client_secret).get_access_token() + if not access_token: + return None + + user_id, tenant_id = get_user_and_tenant_ids_from_access_token(access_token) + return AuthInfo(user_id=user_id, tenant_id=tenant_id) + except (RequestHttpError, HttpUnauthorizedError): + if ctx: + printer.print_exception() + + return None + + +def _try_oidc_authorization( + ctx: 'Context', printer: any, client_id: Optional[str], id_token: Optional[str] +) -> Optional[AuthInfo]: + if not client_id or not id_token: + return None + + try: + access_token = CycodeOidcBasedClient(client_id, id_token).get_access_token() + if not access_token: + return None + + user_id, tenant_id = get_user_and_tenant_ids_from_access_token(access_token) + return AuthInfo(user_id=user_id, tenant_id=tenant_id) + except (RequestHttpError, HttpUnauthorizedError): + if ctx: + printer.print_exception() + + return None diff --git a/cycode/cli/auth/auth_manager.py b/cycode/cli/apps/auth/auth_manager.py similarity index 50% rename from cycode/cli/auth/auth_manager.py rename to cycode/cli/apps/auth/auth_manager.py index da2ca3b4..56a480e4 100644 --- a/cycode/cli/auth/auth_manager.py +++ b/cycode/cli/apps/auth/auth_manager.py @@ -1,103 +1,95 @@ import time import webbrowser -from requests import Request -from typing import Optional +from typing import TYPE_CHECKING from cycode.cli.exceptions.custom_exceptions import AuthProcessError -from cycode.cli.utils.string_utils import generate_random_string, hash_string_to_sha256 from cycode.cli.user_settings.configuration_manager import ConfigurationManager from cycode.cli.user_settings.credentials_manager import CredentialsManager +from cycode.cli.utils.string_utils import generate_random_string, hash_string_to_sha256 from cycode.cyclient.auth_client import AuthClient -from cycode.cyclient.models import ApiToken, ApiTokenGenerationPollingResponse -from cycode.cyclient import logger +from cycode.cyclient.models import ApiTokenGenerationPollingResponse +from cycode.logger import get_logger + +if TYPE_CHECKING: + from cycode.cyclient.models import ApiToken + + +logger = get_logger('Auth Manager') class AuthManager: CODE_VERIFIER_LENGTH = 101 POLLING_WAIT_INTERVAL_IN_SECONDS = 3 POLLING_TIMEOUT_IN_SECONDS = 180 - FAILED_POLLING_STATUS = "Error" - COMPLETED_POLLING_STATUS = "Completed" - - configuration_manager: ConfigurationManager - credentials_manager: CredentialsManager - auth_client: AuthClient + FAILED_POLLING_STATUS = 'Error' + COMPLETED_POLLING_STATUS = 'Completed' - def __init__(self): + def __init__(self) -> None: self.configuration_manager = ConfigurationManager() self.credentials_manager = CredentialsManager() self.auth_client = AuthClient() - def authenticate(self): - logger.debug('generating pkce code pair') + def authenticate(self) -> None: + logger.debug('Generating PKCE code pair') code_challenge, code_verifier = self._generate_pkce_code_pair() - logger.debug('starting authentication session') + logger.debug('Starting authentication session') session_id = self.start_session(code_challenge) - logger.debug('authentication session created, %s', {'session_id': session_id}) + logger.debug('Authentication session created, %s', {'session_id': session_id}) - logger.debug('opening browser and redirecting to cycode login page') + logger.debug('Opening browser and redirecting to Cycode login page') self.redirect_to_login_page(code_challenge, session_id) - logger.debug('starting get api token process') + logger.debug('Getting API token') api_token = self.get_api_token(session_id, code_verifier) - logger.debug('saving get api token') + logger.debug('Saving API token') self.save_api_token(api_token) - def start_session(self, code_challenge: str): + def start_session(self, code_challenge: str) -> str: auth_session = self.auth_client.start_session(code_challenge) return auth_session.session_id - def redirect_to_login_page(self, code_challenge: str, session_id: str): - login_url = self._build_login_url(code_challenge, session_id) + def redirect_to_login_page(self, code_challenge: str, session_id: str) -> None: + login_url = self.auth_client.build_login_url(code_challenge, session_id) webbrowser.open(login_url) - def get_api_token(self, session_id: str, code_verifier: str) -> Optional[ApiToken]: + def get_api_token(self, session_id: str, code_verifier: str) -> 'ApiToken': api_token = self.get_api_token_polling(session_id, code_verifier) if api_token is None: - raise AuthProcessError("getting api token is completed, but the token is missing") + raise AuthProcessError('API token pulling is completed, but the token is missing') return api_token - def get_api_token_polling(self, session_id: str, code_verifier: str) -> Optional[ApiToken]: + def get_api_token_polling(self, session_id: str, code_verifier: str) -> 'ApiToken': end_polling_time = time.time() + self.POLLING_TIMEOUT_IN_SECONDS while time.time() < end_polling_time: - logger.debug('trying to get api token...') + logger.debug('Trying to get API token...') api_token_polling_response = self.auth_client.get_api_token(session_id, code_verifier) if self._is_api_token_process_completed(api_token_polling_response): - logger.debug('get api token process completed') + logger.debug('Got API token process completion response') return api_token_polling_response.api_token if self._is_api_token_process_failed(api_token_polling_response): - logger.debug('get api token process failed') - raise AuthProcessError('error during getting api token') + logger.debug('Got API token process failure response') + raise AuthProcessError('Error while obtaining API token') time.sleep(self.POLLING_WAIT_INTERVAL_IN_SECONDS) - raise AuthProcessError('session expired') - - def save_api_token(self, api_token: ApiToken): - self.credentials_manager.update_credentials_file(api_token.client_id, api_token.secret) + raise AuthProcessError('Timeout while obtaining API token (session expired)') - def _build_login_url(self, code_challenge: str, session_id: str): - app_url = self.configuration_manager.get_cycode_app_url() - login_url = f'{app_url}/account/sign-in' - query_params = { - 'source': 'cycode_cli', - 'code_challenge': code_challenge, - 'session_id': session_id - } - # TODO(MarshalX). Use auth_client instead and don't depend on "requests" lib here - request = Request(url=login_url, params=query_params) - return request.prepare().url + def save_api_token(self, api_token: 'ApiToken') -> None: + self.credentials_manager.update_credentials(api_token.client_id, api_token.secret) - def _generate_pkce_code_pair(self) -> (str, str): + def _generate_pkce_code_pair(self) -> tuple[str, str]: code_verifier = generate_random_string(self.CODE_VERIFIER_LENGTH) code_challenge = hash_string_to_sha256(code_verifier) return code_challenge, code_verifier def _is_api_token_process_completed(self, api_token_polling_response: ApiTokenGenerationPollingResponse) -> bool: - return api_token_polling_response is not None \ - and api_token_polling_response.status == self.COMPLETED_POLLING_STATUS + return ( + api_token_polling_response is not None + and api_token_polling_response.status == self.COMPLETED_POLLING_STATUS + ) def _is_api_token_process_failed(self, api_token_polling_response: ApiTokenGenerationPollingResponse) -> bool: - return api_token_polling_response is not None \ - and api_token_polling_response.status == self.FAILED_POLLING_STATUS + return ( + api_token_polling_response is not None and api_token_polling_response.status == self.FAILED_POLLING_STATUS + ) diff --git a/cycode/cli/apps/auth/models.py b/cycode/cli/apps/auth/models.py new file mode 100644 index 00000000..4b41dd3e --- /dev/null +++ b/cycode/cli/apps/auth/models.py @@ -0,0 +1,6 @@ +from typing import NamedTuple + + +class AuthInfo(NamedTuple): + user_id: str + tenant_id: str diff --git a/cycode/cli/apps/configure/__init__.py b/cycode/cli/apps/configure/__init__.py new file mode 100644 index 00000000..4944a3e3 --- /dev/null +++ b/cycode/cli/apps/configure/__init__.py @@ -0,0 +1,14 @@ +import typer + +from cycode.cli.apps.configure.configure_command import configure_command + +_configure_command_docs = 'https://github.com/cycodehq/cycode-cli/blob/main/README.md#using-the-configure-command' +_configure_command_epilog = f'[bold]Documentation:[/] [link={_configure_command_docs}]{_configure_command_docs}[/link]' + + +app = typer.Typer(no_args_is_help=True) +app.command( + name='configure', + epilog=_configure_command_epilog, + short_help='Initial command to configure your CLI client authentication.', +)(configure_command) diff --git a/cycode/cli/apps/configure/configure_command.py b/cycode/cli/apps/configure/configure_command.py new file mode 100644 index 00000000..1811271c --- /dev/null +++ b/cycode/cli/apps/configure/configure_command.py @@ -0,0 +1,75 @@ +from typing import Optional + +from cycode.cli.apps.configure.consts import CONFIGURATION_MANAGER, CREDENTIALS_MANAGER +from cycode.cli.apps.configure.messages import get_credentials_update_result_message, get_urls_update_result_message +from cycode.cli.apps.configure.prompts import ( + get_api_url_input, + get_app_url_input, + get_client_id_input, + get_client_secret_input, + get_id_token_input, +) +from cycode.cli.console import console + + +def _should_update_value( + old_value: Optional[str], + new_value: Optional[str], +) -> bool: + if not new_value: + return False + + return old_value != new_value + + +def configure_command() -> None: + """:gear: [bold cyan]Configure Cycode CLI settings.[/] + + This command allows you to configure various aspects of the Cycode CLI. + + Configuration options: + * API URL: The base URL for Cycode's API (for on-premise or EU installations) + * APP URL: The base URL for Cycode's web application (for on-premise or EU installations) + * Client ID: Your Cycode client ID for authentication + * Client Secret: Your Cycode client secret for authentication + * ID Token: Your Cycode ID token for authentication + + Example usage: + * `cycode configure`: Start interactive configuration + * `cycode configure --help`: View configuration options + """ + global_config_manager = CONFIGURATION_MANAGER.global_config_file_manager + + current_api_url = global_config_manager.get_api_url() + current_app_url = global_config_manager.get_app_url() + api_url = get_api_url_input(current_api_url) + app_url = get_app_url_input(current_app_url) + + config_updated = False + if _should_update_value(current_api_url, api_url): + global_config_manager.update_api_base_url(api_url) + config_updated = True + if _should_update_value(current_app_url, app_url): + global_config_manager.update_app_base_url(app_url) + config_updated = True + + current_client_id, current_client_secret = CREDENTIALS_MANAGER.get_credentials_from_file() + _, current_id_token = CREDENTIALS_MANAGER.get_oidc_credentials_from_file() + client_id = get_client_id_input(current_client_id) + client_secret = get_client_secret_input(current_client_secret) + id_token = get_id_token_input(current_id_token) + + credentials_updated = False + if _should_update_value(current_client_id, client_id) or _should_update_value(current_client_secret, client_secret): + credentials_updated = True + CREDENTIALS_MANAGER.update_credentials(client_id, client_secret) + + oidc_credentials_updated = False + if _should_update_value(current_client_id, client_id) or _should_update_value(current_id_token, id_token): + oidc_credentials_updated = True + CREDENTIALS_MANAGER.update_oidc_credentials(client_id, id_token) + + if config_updated: + console.print(get_urls_update_result_message()) + if credentials_updated or oidc_credentials_updated: + console.print(get_credentials_update_result_message()) diff --git a/cycode/cli/apps/configure/consts.py b/cycode/cli/apps/configure/consts.py new file mode 100644 index 00000000..15c9b7a5 --- /dev/null +++ b/cycode/cli/apps/configure/consts.py @@ -0,0 +1,19 @@ +from cycode.cli import config, consts +from cycode.cli.user_settings.configuration_manager import ConfigurationManager +from cycode.cli.user_settings.credentials_manager import CredentialsManager + +URLS_UPDATED_SUCCESSFULLY_MESSAGE = 'Successfully configured Cycode URLs! Saved to: {filename}' +URLS_ARE_SET_IN_ENVIRONMENT_VARIABLES_MESSAGE = ( + 'Note that the URLs (APP and API) that already exist in environment variables ' + f'({consts.CYCODE_API_URL_ENV_VAR_NAME} and {consts.CYCODE_APP_URL_ENV_VAR_NAME}) ' + 'take precedent over these URLs; either update or remove the environment variables.' +) +CREDENTIALS_UPDATED_SUCCESSFULLY_MESSAGE = 'Successfully configured CLI credentials! Saved to: {filename}' +CREDENTIALS_ARE_SET_IN_ENVIRONMENT_VARIABLES_MESSAGE = ( + 'Note that the credentials that already exist in environment variables ' + f'({config.CYCODE_CLIENT_ID_ENV_VAR_NAME} and {config.CYCODE_CLIENT_SECRET_ENV_VAR_NAME}) ' + 'take precedent over these credentials; either update or remove the environment variables.' +) + +CREDENTIALS_MANAGER = CredentialsManager() +CONFIGURATION_MANAGER = ConfigurationManager() diff --git a/cycode/cli/apps/configure/messages.py b/cycode/cli/apps/configure/messages.py new file mode 100644 index 00000000..36ce807b --- /dev/null +++ b/cycode/cli/apps/configure/messages.py @@ -0,0 +1,37 @@ +from cycode.cli.apps.configure.consts import ( + CONFIGURATION_MANAGER, + CREDENTIALS_ARE_SET_IN_ENVIRONMENT_VARIABLES_MESSAGE, + CREDENTIALS_MANAGER, + CREDENTIALS_UPDATED_SUCCESSFULLY_MESSAGE, + URLS_ARE_SET_IN_ENVIRONMENT_VARIABLES_MESSAGE, + URLS_UPDATED_SUCCESSFULLY_MESSAGE, +) + + +def _are_credentials_exist_in_environment_variables() -> bool: + client_id, client_secret = CREDENTIALS_MANAGER.get_credentials_from_environment_variables() + return any([client_id, client_secret]) + + +def get_credentials_update_result_message() -> str: + success_message = CREDENTIALS_UPDATED_SUCCESSFULLY_MESSAGE.format(filename=CREDENTIALS_MANAGER.get_filename()) + if _are_credentials_exist_in_environment_variables(): + return f'{success_message}. {CREDENTIALS_ARE_SET_IN_ENVIRONMENT_VARIABLES_MESSAGE}' + + return success_message + + +def _are_urls_exist_in_environment_variables() -> bool: + api_url = CONFIGURATION_MANAGER.get_api_url_from_environment_variables() + app_url = CONFIGURATION_MANAGER.get_app_url_from_environment_variables() + return any([api_url, app_url]) + + +def get_urls_update_result_message() -> str: + success_message = URLS_UPDATED_SUCCESSFULLY_MESSAGE.format( + filename=CONFIGURATION_MANAGER.global_config_file_manager.get_filename() + ) + if _are_urls_exist_in_environment_variables(): + return f'{success_message}. {URLS_ARE_SET_IN_ENVIRONMENT_VARIABLES_MESSAGE}' + + return success_message diff --git a/cycode/cli/apps/configure/prompts.py b/cycode/cli/apps/configure/prompts.py new file mode 100644 index 00000000..63fae12c --- /dev/null +++ b/cycode/cli/apps/configure/prompts.py @@ -0,0 +1,59 @@ +from typing import Optional + +import typer + +from cycode.cli import consts +from cycode.cli.utils.string_utils import obfuscate_text + + +def get_client_id_input(current_client_id: Optional[str]) -> Optional[str]: + prompt_text = 'Cycode Client ID' + + prompt_suffix = ' []: ' + if current_client_id: + prompt_suffix = f' [{obfuscate_text(current_client_id)}]: ' + + new_client_id = typer.prompt(text=prompt_text, prompt_suffix=prompt_suffix, default='', show_default=False) + return new_client_id or current_client_id + + +def get_client_secret_input(current_client_secret: Optional[str]) -> Optional[str]: + prompt_text = 'Cycode Client Secret' + + prompt_suffix = ' []: ' + if current_client_secret: + prompt_suffix = f' [{obfuscate_text(current_client_secret)}]: ' + + new_client_secret = typer.prompt(text=prompt_text, prompt_suffix=prompt_suffix, default='', show_default=False) + return new_client_secret or current_client_secret + + +def get_app_url_input(current_app_url: Optional[str]) -> str: + prompt_text = 'Cycode APP URL' + + default = consts.DEFAULT_CYCODE_APP_URL + if current_app_url: + default = current_app_url + + return typer.prompt(text=prompt_text, default=default, type=str) + + +def get_api_url_input(current_api_url: Optional[str]) -> str: + prompt_text = 'Cycode API URL' + + default = consts.DEFAULT_CYCODE_API_URL + if current_api_url: + default = current_api_url + + return typer.prompt(text=prompt_text, default=default, type=str) + + +def get_id_token_input(current_id_token: Optional[str]) -> Optional[str]: + prompt_text = 'Cycode ID Token' + + prompt_suffix = ' []: ' + if current_id_token: + prompt_suffix = f' [{obfuscate_text(current_id_token)}]: ' + + new_id_token = typer.prompt(text=prompt_text, prompt_suffix=prompt_suffix, default='', show_default=False) + return new_id_token or current_id_token diff --git a/cycode/cli/apps/ignore/__init__.py b/cycode/cli/apps/ignore/__init__.py new file mode 100644 index 00000000..e6573b69 --- /dev/null +++ b/cycode/cli/apps/ignore/__init__.py @@ -0,0 +1,6 @@ +import typer + +from cycode.cli.apps.ignore.ignore_command import ignore_command + +app = typer.Typer(no_args_is_help=True) +app.command(name='ignore', short_help='Ignores a specific value, path or rule ID.')(ignore_command) diff --git a/cycode/cli/apps/ignore/ignore_command.py b/cycode/cli/apps/ignore/ignore_command.py new file mode 100644 index 00000000..c65197c3 --- /dev/null +++ b/cycode/cli/apps/ignore/ignore_command.py @@ -0,0 +1,158 @@ +import re +from typing import Annotated, Optional + +import click +import typer + +from cycode.cli import consts +from cycode.cli.cli_types import ScanTypeOption +from cycode.cli.config import configuration_manager +from cycode.cli.logger import logger +from cycode.cli.utils.path_utils import get_absolute_path, is_path_exists +from cycode.cli.utils.string_utils import hash_string_to_sha256 + +_FILTER_BY_RICH_HELP_PANEL = 'Filter options' +_SECRETS_FILTER_BY_RICH_HELP_PANEL = 'Secrets filter options' +_SCA_FILTER_BY_RICH_HELP_PANEL = 'SCA filter options' + + +def _is_package_pattern_valid(package: str) -> bool: + return re.search('^[^@]+@[^@]+$', package) is not None + + +def ignore_command( # noqa: C901 + by_path: Annotated[ + Optional[str], + typer.Option( + help='Ignore a specific file or directory while scanning.', + show_default=False, + rich_help_panel=_FILTER_BY_RICH_HELP_PANEL, + ), + ] = None, + by_rule: Annotated[ + Optional[str], + typer.Option( + help='Ignore scanning a specific Secrets rule ID or IaC rule ID.', + show_default=False, + rich_help_panel=_FILTER_BY_RICH_HELP_PANEL, + ), + ] = None, + by_value: Annotated[ + Optional[str], + typer.Option( + help='Ignore a specific value.', + show_default=False, + rich_help_panel=_SECRETS_FILTER_BY_RICH_HELP_PANEL, + ), + ] = None, + by_sha: Annotated[ + Optional[str], + typer.Option( + help='Ignore a specific SHA512 representation of a string.', + show_default=False, + rich_help_panel=_SECRETS_FILTER_BY_RICH_HELP_PANEL, + ), + ] = None, + by_package: Annotated[ + Optional[str], + typer.Option( + help='Ignore scanning a specific package version. Expected pattern: [cyan]name@version[/].', + show_default=False, + rich_help_panel=_SCA_FILTER_BY_RICH_HELP_PANEL, + ), + ] = None, + by_cve: Annotated[ + Optional[str], + typer.Option( + help='Ignore scanning a specific CVE. Expected pattern: [cyan]CVE-YYYY-NNN[/].', + show_default=False, + rich_help_panel=_SCA_FILTER_BY_RICH_HELP_PANEL, + ), + ] = None, + scan_type: Annotated[ + ScanTypeOption, + typer.Option( + '--scan-type', + '-t', + help='Specify the type of scan you wish to execute.', + case_sensitive=False, + ), + ] = ScanTypeOption.SECRET, + is_global: Annotated[ + bool, typer.Option('--global', '-g', help='Add an ignore rule to the global CLI config.') + ] = False, +) -> None: + """:no_entry: [bold cyan]Ignore specific findings or paths in scans.[/] + + This command allows you to exclude specific items from Cycode scans, including: + * Paths: Exclude specific files or directories + * Rules: Ignore specific security rules + * Values: Exclude specific sensitive values + * Packages: Ignore specific package versions + * CVEs: Exclude specific vulnerabilities + + Example usage: + * `cycode ignore --by-path .env`: Ignore the tests directory + * `cycode ignore --by-rule GUID`: Ignore rule with the specified GUID + * `cycode ignore --by-package lodash@4.17.21`: Ignore lodash version 4.17.21 + """ + all_by_values = [by_value, by_sha, by_path, by_rule, by_package, by_cve] + if all(by is None for by in all_by_values): + raise click.ClickException('Ignore by type is missing') + if len([by for by in all_by_values if by is not None]) != 1: + raise click.ClickException('You must specify only one ignore by type') + + if any(by is not None for by in [by_value, by_sha]) and scan_type != consts.SECRET_SCAN_TYPE: + raise click.ClickException('This exclude is supported only for Secret scan type') + if (by_cve or by_package) and scan_type != consts.SCA_SCAN_TYPE: + raise click.ClickException('This exclude is supported only for SCA scan type') + + # only one of the by values must be set + # at least one of the by values must be set + exclusion_type = exclusion_value = None + + if by_value: + exclusion_type = consts.EXCLUSIONS_BY_VALUE_SECTION_NAME + exclusion_value = hash_string_to_sha256(by_value) + + if by_sha: + exclusion_type = consts.EXCLUSIONS_BY_SHA_SECTION_NAME + exclusion_value = by_sha + + if by_path: + absolute_path = get_absolute_path(by_path) + if not is_path_exists(absolute_path): + raise click.ClickException('The provided path to ignore by does not exist') + + exclusion_type = consts.EXCLUSIONS_BY_PATH_SECTION_NAME + exclusion_value = get_absolute_path(absolute_path) + + if by_rule: + exclusion_type = consts.EXCLUSIONS_BY_RULE_SECTION_NAME + exclusion_value = by_rule + + if by_package: + if not _is_package_pattern_valid(by_package): + raise click.ClickException('wrong package pattern. should be name@version.') + + exclusion_type = consts.EXCLUSIONS_BY_PACKAGE_SECTION_NAME + exclusion_value = by_package + + if by_cve: + exclusion_type = consts.EXCLUSIONS_BY_CVE_SECTION_NAME + exclusion_value = by_cve + + if not exclusion_type or not exclusion_value: + # should never happen + raise click.ClickException('Invalid ignore by type') + + configuration_scope = 'global' if is_global else 'local' + logger.debug( + 'Adding ignore rule, %s', + { + 'configuration_scope': configuration_scope, + 'exclusion_type': exclusion_type, + 'exclusion_value': exclusion_value, + }, + ) + configuration_manager.add_exclusion(configuration_scope, str(scan_type), exclusion_type, exclusion_value) diff --git a/cycode/cli/apps/mcp/__init__.py b/cycode/cli/apps/mcp/__init__.py new file mode 100644 index 00000000..fd328845 --- /dev/null +++ b/cycode/cli/apps/mcp/__init__.py @@ -0,0 +1,14 @@ +import typer + +from cycode.cli.apps.mcp.mcp_command import mcp_command + +app = typer.Typer() + +_mcp_command_docs = 'https://github.com/cycodehq/cycode-cli/blob/main/README.md#mcp-command-experiment' +_mcp_command_epilog = f'[bold]Documentation:[/] [link={_mcp_command_docs}]{_mcp_command_docs}[/link]' + +app.command( + name='mcp', + short_help='[EXPERIMENT] Start the Cycode MCP (Model Context Protocol) server.', + epilog=_mcp_command_epilog, +)(mcp_command) diff --git a/cycode/cli/apps/mcp/mcp_command.py b/cycode/cli/apps/mcp/mcp_command.py new file mode 100644 index 00000000..39bcce40 --- /dev/null +++ b/cycode/cli/apps/mcp/mcp_command.py @@ -0,0 +1,387 @@ +import asyncio +import json +import logging +import os +import shutil +import sys +import tempfile +import uuid +from typing import Annotated, Any + +import typer +from pathvalidate import sanitize_filepath +from pydantic import Field + +from cycode.cli.cli_types import McpTransportOption, ScanTypeOption +from cycode.logger import LoggersManager, get_logger + +try: + from mcp.server.fastmcp import FastMCP + from mcp.server.fastmcp.tools import Tool +except ImportError: + raise ImportError( + 'Cycode MCP is not supported for your Python version. MCP support requires Python 3.10 or higher.' + ) from None + + +_logger = get_logger('Cycode MCP') + +_DEFAULT_RUN_COMMAND_TIMEOUT = 10 * 60 + +_FILES_TOOL_FIELD = Field(description='Files to scan, mapping file paths to their content') + + +def _is_debug_mode() -> bool: + return LoggersManager.global_logging_level == logging.DEBUG + + +def _gen_random_id() -> str: + return uuid.uuid4().hex + + +def _get_current_executable() -> str: + """Get the current executable path for spawning subprocess.""" + if getattr(sys, 'frozen', False): # pyinstaller bundle + return sys.executable + + return 'cycode' + + +async def _run_cycode_command(*args: str, timeout: int = _DEFAULT_RUN_COMMAND_TIMEOUT) -> dict[str, Any]: + """Run a cycode command asynchronously and return the parsed result. + + Args: + *args: Command arguments to append after 'cycode -o json' + timeout: Timeout in seconds (default 5 minutes) + + Returns: + Dictionary containing the parsed JSON result or error information + """ + verbose = ['-v'] if _is_debug_mode() else [] + cmd_args = [_get_current_executable(), *verbose, '-o', 'json', *list(args)] + _logger.debug('Running Cycode CLI command: %s', ' '.join(cmd_args)) + + try: + process = await asyncio.create_subprocess_exec( + *cmd_args, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE + ) + + stdout, stderr = await asyncio.wait_for(process.communicate(), timeout=timeout) + stdout_str = stdout.decode('UTF-8', errors='replace') if stdout else '' + stderr_str = stderr.decode('UTF-8', errors='replace') if stderr else '' + + if _is_debug_mode(): # redirect debug output + sys.stderr.write(stderr_str) + + if not stdout_str: + return {'error': 'No output from command', 'stderr': stderr_str, 'returncode': process.returncode} + + try: + return json.loads(stdout_str) + except json.JSONDecodeError: + return { + 'error': 'Failed to parse JSON output', + 'stdout': stdout_str, + 'stderr': stderr_str, + 'returncode': process.returncode, + } + except asyncio.TimeoutError: + return {'error': f'Command timeout after {timeout} seconds'} + except Exception as e: + return {'error': f'Failed to run command: {e!s}'} + + +def _sanitize_file_path(file_path: str) -> str: + """Sanitize file path to prevent path traversal and other security issues. + + Args: + file_path: The file path to sanitize + + Returns: + Sanitized file path safe for use in temporary directory + + Raises: + ValueError: If the path is invalid or potentially dangerous + """ + if not file_path or not isinstance(file_path, str): + raise ValueError('File path must be a non-empty string') + + return sanitize_filepath(file_path, platform='auto', validate_after_sanitize=True) + + +class _TempFilesManager: + """Context manager for creating and cleaning up temporary files. + + Creates a temporary directory structure that preserves original file paths + inside a call_id as a suffix. Automatically cleans up all files and directories + when exiting the context. + """ + + def __init__(self, files_content: dict[str, str], call_id: str) -> None: + self.files_content = files_content + self.call_id = call_id + self.temp_base_dir = None + self.temp_files = [] + + def __enter__(self) -> list[str]: + self.temp_base_dir = tempfile.mkdtemp(prefix='cycode_mcp_', suffix=self.call_id) + _logger.debug('Creating temporary files in directory: %s', self.temp_base_dir) + + for file_path, content in self.files_content.items(): + try: + sanitized_path = _sanitize_file_path(file_path) + temp_file_path = os.path.join(self.temp_base_dir, sanitized_path) + + # Ensure the normalized path is still within our temp directory + normalized_temp_path = os.path.normpath(temp_file_path) + normalized_base_path = os.path.normpath(self.temp_base_dir) + if not normalized_temp_path.startswith(normalized_base_path + os.sep): + raise ValueError(f'Path escapes temporary directory: {file_path}') + + os.makedirs(os.path.dirname(temp_file_path), exist_ok=True) + + _logger.debug('Creating temp file: %s (from: %s)', temp_file_path, file_path) + with open(temp_file_path, 'w', encoding='UTF-8') as f: + f.write(content) + + self.temp_files.append(temp_file_path) + except ValueError as e: + _logger.error('Invalid file path rejected: %s - %s', file_path, str(e)) + continue + except Exception as e: + _logger.error('Failed to create temp file for %s: %s', file_path, str(e)) + continue + + if not self.temp_files: + raise ValueError('No valid files provided after sanitization') + + return self.temp_files + + def __exit__(self, *_) -> None: + if self.temp_base_dir and os.path.exists(self.temp_base_dir): + _logger.debug('Removing temp directory recursively: %s', self.temp_base_dir) + shutil.rmtree(self.temp_base_dir, ignore_errors=True) + + +async def _run_cycode_scan(scan_type: ScanTypeOption, temp_files: list[str]) -> dict[str, Any]: + """Run cycode scan command and return the result.""" + return await _run_cycode_command(*['scan', '-t', str(scan_type), 'path', *temp_files]) + + +async def _run_cycode_status() -> dict[str, Any]: + """Run cycode status command and return the result.""" + return await _run_cycode_command('status') + + +async def _cycode_scan_tool(scan_type: ScanTypeOption, files: dict[str, str] = _FILES_TOOL_FIELD) -> str: + _tool_call_id = _gen_random_id() + _logger.info('Scan tool called, %s', {'scan_type': scan_type, 'call_id': _tool_call_id}) + + if not files: + _logger.error('No files provided for scan') + return json.dumps({'error': 'No files provided'}) + + try: + with _TempFilesManager(files, _tool_call_id) as temp_files: + original_count = len(files) + processed_count = len(temp_files) + + if processed_count < original_count: + _logger.warning( + 'Some files were rejected during sanitization, %s', + { + 'scan_type': scan_type, + 'original_count': original_count, + 'processed_count': processed_count, + 'call_id': _tool_call_id, + }, + ) + + _logger.info( + 'Running Cycode scan, %s', + {'scan_type': scan_type, 'files_count': processed_count, 'call_id': _tool_call_id}, + ) + result = await _run_cycode_scan(scan_type, temp_files) + + _logger.info('Scan completed, %s', {'scan_type': scan_type, 'call_id': _tool_call_id}) + return json.dumps(result, indent=2) + except ValueError as e: + _logger.error('Invalid input files, %s', {'scan_type': scan_type, 'call_id': _tool_call_id, 'error': str(e)}) + return json.dumps({'error': f'Invalid input files: {e!s}'}, indent=2) + except Exception as e: + _logger.error('Scan failed, %s', {'scan_type': scan_type, 'call_id': _tool_call_id, 'error': str(e)}) + return json.dumps({'error': f'Scan failed: {e!s}'}, indent=2) + + +async def cycode_secret_scan(files: dict[str, str] = _FILES_TOOL_FIELD) -> str: + """Scan files for hardcoded secrets. + + Use this tool when you need to: + - scan code for hardcoded secrets, API keys, passwords, tokens + - verify that code doesn't contain exposed credentials + - detect potential security vulnerabilities from secret exposure + + Args: + files: Dictionary mapping file paths to their content + + Returns: + JSON string containing scan results and any secrets found + """ + return await _cycode_scan_tool(ScanTypeOption.SECRET, files) + + +async def cycode_sca_scan(files: dict[str, str] = _FILES_TOOL_FIELD) -> str: + """Scan files for Software Composition Analysis (SCA) - vulnerabilities and license issues. + + Use this tool when you need to: + - scan dependencies for known security vulnerabilities + - check for license compliance issues + - analyze third-party component risks + - verify software supply chain security + - review package.json, requirements.txt, pom.xml and other dependency files + + Important: + You must also include lock files (like package-lock.json, Pipfile.lock, etc.) to get accurate results. + You must provide manifest and lock files together. + + Args: + files: Dictionary mapping file paths to their content + + Returns: + JSON string containing scan results, vulnerabilities, and license issues found + """ + return await _cycode_scan_tool(ScanTypeOption.SCA, files) + + +async def cycode_iac_scan(files: dict[str, str] = _FILES_TOOL_FIELD) -> str: + """Scan files for Infrastructure as Code (IaC) misconfigurations. + + Use this tool when you need to: + - scan Terraform, CloudFormation, Kubernetes YAML files + - check for cloud security misconfigurations + - verify infrastructure compliance and best practices + - detect potential security issues in infrastructure definitions + - review Docker files for security issues + + Args: + files: Dictionary mapping file paths to their content + + Returns: + JSON string containing scan results and any misconfigurations found + """ + return await _cycode_scan_tool(ScanTypeOption.IAC, files) + + +async def cycode_sast_scan(files: dict[str, str] = _FILES_TOOL_FIELD) -> str: + """Scan files for Static Application Security Testing (SAST) - code quality and security flaws. + + Use this tool when you need to: + - scan source code for security vulnerabilities + - detect code quality issues and potential bugs + - check for insecure coding practices + - verify code follows security best practices + - find SQL injection, XSS, and other application security issues + + Args: + files: Dictionary mapping file paths to their content + + Returns: + JSON string containing scan results and any security flaws found + """ + return await _cycode_scan_tool(ScanTypeOption.SAST, files) + + +async def cycode_status() -> str: + """Get Cycode CLI version, authentication status, and configuration information. + + Use this tool when you need to: + - verify Cycode CLI is properly configured + - check authentication status + - get CLI version information + - troubleshoot setup issues + - confirm service connectivity + + Returns: + JSON string containing CLI status, version, and configuration details + """ + _tool_call_id = _gen_random_id() + _logger.info('Status tool called') + + try: + _logger.info('Running Cycode status check, %s', {'call_id': _tool_call_id}) + result = await _run_cycode_status() + _logger.info('Status check completed, %s', {'call_id': _tool_call_id}) + + return json.dumps(result, indent=2) + except Exception as e: + _logger.error('Status check failed, %s', {'call_id': _tool_call_id, 'error': str(e)}) + return json.dumps({'error': f'Status check failed: {e!s}'}, indent=2) + + +def _create_mcp_server(host: str, port: int) -> FastMCP: + """Create and configure the MCP server.""" + tools = [ + Tool.from_function(cycode_status), + Tool.from_function(cycode_secret_scan), + Tool.from_function(cycode_sca_scan), + Tool.from_function(cycode_iac_scan), + Tool.from_function(cycode_sast_scan), + ] + _logger.info('Creating MCP server with tools: %s', [tool.name for tool in tools]) + return FastMCP( + 'cycode', + tools=tools, + host=host, + port=port, + debug=_is_debug_mode(), + log_level='DEBUG' if _is_debug_mode() else 'INFO', + ) + + +def _run_mcp_server(transport: McpTransportOption, host: str, port: int) -> None: + """Run the MCP server using transport.""" + mcp = _create_mcp_server(host, port) + mcp.run(transport=str(transport)) # type: ignore[arg-type] + + +def mcp_command( + transport: Annotated[ + McpTransportOption, + typer.Option( + '--transport', + '-t', + case_sensitive=False, + help='Transport type for the MCP server.', + ), + ] = McpTransportOption.STDIO, + host: str = typer.Option( + '127.0.0.1', + '--host', + '-H', + help='Host address to bind the server (used only for non stdio transport).', + ), + port: int = typer.Option( + 8000, + '--port', + '-p', + help='Port number to bind the server (used only for non stdio transport).', + ), +) -> None: + """:robot: Start the Cycode MCP (Model Context Protocol) server. + + The MCP server provides tools for scanning code with Cycode CLI: + - cycode_secret_scan: Scan for hardcoded secrets + - cycode_sca_scan: Software Composition Analysis scanning + - cycode_iac_scan: Infrastructure as Code scanning + - cycode_sast_scan: Static Application Security Testing scanning + - cycode_status: Get Cycode CLI status (version, auth status) and configuration + + Examples: + cycode mcp # Start with default transport (stdio) + cycode mcp -t sse -p 8080 # Start with Server-Sent Events (SSE) transport on port 8080 + """ + try: + _run_mcp_server(transport, host, port) + except Exception as e: + _logger.error('MCP server error', exc_info=e) + raise typer.Exit(1) from e diff --git a/cycode/cli/apps/report/__init__.py b/cycode/cli/apps/report/__init__.py new file mode 100644 index 00000000..751157a4 --- /dev/null +++ b/cycode/cli/apps/report/__init__.py @@ -0,0 +1,8 @@ +import typer + +from cycode.cli.apps.report import sbom +from cycode.cli.apps.report.report_command import report_command + +app = typer.Typer(name='report', no_args_is_help=True) +app.callback(short_help='Generate report. You`ll need to specify which report type to perform as SBOM.')(report_command) +app.add_typer(sbom.app) diff --git a/cycode/cli/apps/report/report_command.py b/cycode/cli/apps/report/report_command.py new file mode 100644 index 00000000..ba19be1c --- /dev/null +++ b/cycode/cli/apps/report/report_command.py @@ -0,0 +1,13 @@ +import typer + +from cycode.cli.utils.progress_bar import SBOM_REPORT_PROGRESS_BAR_SECTIONS, get_progress_bar + + +def report_command(ctx: typer.Context) -> int: + """:bar_chart: [bold cyan]Generate security reports.[/] + + Example usage: + * `cycode report sbom`: Generate SBOM report + """ + ctx.obj['progress_bar'] = get_progress_bar(hidden=False, sections=SBOM_REPORT_PROGRESS_BAR_SECTIONS) + return 1 diff --git a/cycode/cli/apps/report/sbom/__init__.py b/cycode/cli/apps/report/sbom/__init__.py new file mode 100644 index 00000000..77d081e8 --- /dev/null +++ b/cycode/cli/apps/report/sbom/__init__.py @@ -0,0 +1,15 @@ +import typer + +from cycode.cli.apps.report.sbom.path.path_command import path_command +from cycode.cli.apps.report.sbom.repository_url.repository_url_command import repository_url_command +from cycode.cli.apps.report.sbom.sbom_command import sbom_command + +app = typer.Typer(name='sbom') +app.callback(short_help='Generate SBOM report for remote repository by url or local directory by path.')(sbom_command) +app.command(name='path', short_help='Generate SBOM report for provided path in the command.')(path_command) +app.command(name='repository-url', short_help='Generate SBOM report for provided repository URI in the command.')( + repository_url_command +) + +# backward compatibility +app.command(hidden=True, name='repository_url')(repository_url_command) diff --git a/cycode/cli/apps/report/sbom/common.py b/cycode/cli/apps/report/sbom/common.py new file mode 100644 index 00000000..067a9fa6 --- /dev/null +++ b/cycode/cli/apps/report/sbom/common.py @@ -0,0 +1,94 @@ +import pathlib +import time +from platform import platform +from typing import TYPE_CHECKING, Optional + +from cycode.cli import consts +from cycode.cli.apps.report.sbom.sbom_report_file import SbomReportFile +from cycode.cli.config import configuration_manager +from cycode.cli.exceptions.custom_exceptions import ReportAsyncError +from cycode.cli.logger import logger +from cycode.cli.utils.progress_bar import SbomReportProgressBarSection +from cycode.cyclient.models import ReportExecutionSchema + +if TYPE_CHECKING: + from cycode.cli.utils.progress_bar import BaseProgressBar + from cycode.cyclient.report_client import ReportClient + + +def _poll_report_execution_until_completed( + progress_bar: 'BaseProgressBar', + client: 'ReportClient', + report_execution_id: int, + polling_timeout: Optional[int] = None, +) -> ReportExecutionSchema: + if polling_timeout is None: + polling_timeout = configuration_manager.get_report_polling_timeout_in_seconds() + + end_polling_time = time.time() + polling_timeout + while time.time() < end_polling_time: + report_execution = client.get_report_execution(report_execution_id) + report_label = report_execution.error_message or report_execution.status_message + + progress_bar.update_right_side_label(report_label) + + if report_execution.status == consts.REPORT_STATUS_COMPLETED: + return report_execution + + if report_execution.status == consts.REPORT_STATUS_ERROR: + raise ReportAsyncError(f'Error occurred while trying to generate report: {report_label}') + + time.sleep(consts.REPORT_POLLING_WAIT_INTERVAL_IN_SECONDS) + + raise ReportAsyncError(f'Timeout exceeded while waiting for report to complete. Timeout: {polling_timeout} sec.') + + +def send_report_feedback( + client: 'ReportClient', + start_scan_time: float, + report_type: str, + report_command_type: str, + request_report_parameters: dict, + report_execution_id: int, + error_message: Optional[str] = None, + request_zip_file_size: Optional[int] = None, + **kwargs, +) -> None: + try: + request_report_parameters.update(kwargs) + + end_scan_time = time.time() + scan_status = { + 'report_type': report_type, + 'report_command_type': report_command_type, + 'request_report_parameters': request_report_parameters, + 'operation_system': platform(), + 'error_message': error_message, + 'execution_time': int(end_scan_time - start_scan_time), + 'request_zip_file_size': request_zip_file_size, + } + + client.report_status(report_execution_id, scan_status) + except Exception as e: + logger.debug('Failed to send report feedback', exc_info=e) + + +def create_sbom_report( + progress_bar: 'BaseProgressBar', + client: 'ReportClient', + report_execution_id: int, + output_file: Optional[pathlib.Path], + output_format: str, +) -> None: + report_execution = _poll_report_execution_until_completed(progress_bar, client, report_execution_id) + + progress_bar.set_section_length(SbomReportProgressBarSection.GENERATION) + + report_path = report_execution.storage_details.path + report_content = client.get_file_content(report_path) + + progress_bar.set_section_length(SbomReportProgressBarSection.RECEIVE_REPORT) + progress_bar.stop() + + sbom_report = SbomReportFile(report_path, output_format, output_file) + sbom_report.write(report_content) diff --git a/cycode/cli/helpers/__init__.py b/cycode/cli/apps/report/sbom/path/__init__.py similarity index 100% rename from cycode/cli/helpers/__init__.py rename to cycode/cli/apps/report/sbom/path/__init__.py diff --git a/cycode/cli/apps/report/sbom/path/path_command.py b/cycode/cli/apps/report/sbom/path/path_command.py new file mode 100644 index 00000000..a3ffa578 --- /dev/null +++ b/cycode/cli/apps/report/sbom/path/path_command.py @@ -0,0 +1,87 @@ +import time +from pathlib import Path +from typing import Annotated + +import typer + +from cycode.cli import consts +from cycode.cli.apps.report.sbom.common import create_sbom_report, send_report_feedback +from cycode.cli.apps.sca_options import ( + GradleAllSubProjectsOption, + MavenSettingsFileOption, + NoRestoreOption, + apply_sca_restore_options_to_context, +) +from cycode.cli.exceptions.handle_report_sbom_errors import handle_report_exception +from cycode.cli.files_collector.path_documents import get_relevant_documents +from cycode.cli.files_collector.sca.sca_file_collector import add_sca_dependencies_tree_documents_if_needed +from cycode.cli.files_collector.zip_documents import zip_documents +from cycode.cli.utils.get_api_client import get_report_cycode_client +from cycode.cli.utils.progress_bar import SbomReportProgressBarSection +from cycode.cli.utils.scan_utils import is_cycodeignore_allowed_by_scan_config + + +def path_command( + ctx: typer.Context, + path: Annotated[ + Path, + typer.Argument(exists=True, resolve_path=True, help='Path to generate SBOM report for.', show_default=False), + ], + no_restore: NoRestoreOption = False, + gradle_all_sub_projects: GradleAllSubProjectsOption = False, + maven_settings_file: MavenSettingsFileOption = None, +) -> None: + apply_sca_restore_options_to_context(ctx, no_restore, gradle_all_sub_projects, maven_settings_file) + + client = get_report_cycode_client(ctx) + report_parameters = ctx.obj['report_parameters'] + output_format = report_parameters.output_format + output_file = ctx.obj['output_file'] + + progress_bar = ctx.obj['progress_bar'] + progress_bar.start() + + start_scan_time = time.time() + report_execution_id = -1 + + try: + documents = get_relevant_documents( + progress_bar, + SbomReportProgressBarSection.PREPARE_LOCAL_FILES, + consts.SCA_SCAN_TYPE, + (str(path),), + is_cycodeignore_allowed=is_cycodeignore_allowed_by_scan_config(ctx), + ) + # TODO(MarshalX): combine perform_pre_scan_documents_actions with get_relevant_document. + # unhardcode usage of context in perform_pre_scan_documents_actions + add_sca_dependencies_tree_documents_if_needed(ctx, consts.SCA_SCAN_TYPE, documents) + + zipped_documents = zip_documents(consts.SCA_SCAN_TYPE, documents) + report_execution = client.request_sbom_report_execution(report_parameters, zip_file=zipped_documents) + report_execution_id = report_execution.id + + create_sbom_report(progress_bar, client, report_execution_id, output_file, output_format) + + send_report_feedback( + client=client, + start_scan_time=start_scan_time, + report_type='SBOM', + report_command_type='path', + request_report_parameters=report_parameters.to_dict(without_entity_type=False), + report_execution_id=report_execution_id, + request_zip_file_size=zipped_documents.size, + ) + except Exception as e: + progress_bar.stop() + + send_report_feedback( + client=client, + start_scan_time=start_scan_time, + report_type='SBOM', + report_command_type='path', + request_report_parameters=report_parameters.to_dict(without_entity_type=False), + report_execution_id=report_execution_id, + error_message=str(e), + ) + + handle_report_exception(ctx, e) diff --git a/cycode/cli/helpers/maven/__init__.py b/cycode/cli/apps/report/sbom/repository_url/__init__.py similarity index 100% rename from cycode/cli/helpers/maven/__init__.py rename to cycode/cli/apps/report/sbom/repository_url/__init__.py diff --git a/cycode/cli/apps/report/sbom/repository_url/repository_url_command.py b/cycode/cli/apps/report/sbom/repository_url/repository_url_command.py new file mode 100644 index 00000000..2b208ea2 --- /dev/null +++ b/cycode/cli/apps/report/sbom/repository_url/repository_url_command.py @@ -0,0 +1,66 @@ +import time +from typing import Annotated + +import typer + +from cycode.cli.apps.report.sbom.common import create_sbom_report, send_report_feedback +from cycode.cli.exceptions.handle_report_sbom_errors import handle_report_exception +from cycode.cli.utils.get_api_client import get_report_cycode_client +from cycode.cli.utils.progress_bar import SbomReportProgressBarSection +from cycode.cli.utils.url_utils import sanitize_repository_url +from cycode.logger import get_logger + +logger = get_logger('Repository URL Command') + + +def repository_url_command( + ctx: typer.Context, + uri: Annotated[str, typer.Argument(help='Repository URL to generate SBOM report for.', show_default=False)], +) -> None: + progress_bar = ctx.obj['progress_bar'] + progress_bar.start() + progress_bar.set_section_length(SbomReportProgressBarSection.PREPARE_LOCAL_FILES) + + client = get_report_cycode_client(ctx) + report_parameters = ctx.obj['report_parameters'] + output_file = ctx.obj['output_file'] + output_format = report_parameters.output_format + + start_scan_time = time.time() + report_execution_id = -1 + + # Sanitize repository URL to remove any embedded credentials/tokens before sending to API + sanitized_uri = sanitize_repository_url(uri) + if sanitized_uri != uri: + logger.debug('Sanitized repository URL to remove credentials') + + try: + report_execution = client.request_sbom_report_execution(report_parameters, repository_url=sanitized_uri) + report_execution_id = report_execution.id + + create_sbom_report(progress_bar, client, report_execution_id, output_file, output_format) + + send_report_feedback( + client=client, + start_scan_time=start_scan_time, + report_type='SBOM', + report_command_type='repository_url', + request_report_parameters=report_parameters.to_dict(without_entity_type=False), + report_execution_id=report_execution_id, + repository_uri=uri, + ) + except Exception as e: + progress_bar.stop() + + send_report_feedback( + client=client, + start_scan_time=start_scan_time, + report_type='SBOM', + report_command_type='repository_url', + request_report_parameters=report_parameters.to_dict(without_entity_type=False), + report_execution_id=report_execution_id, + error_message=str(e), + repository_uri=uri, + ) + + handle_report_exception(ctx, e) diff --git a/cycode/cli/apps/report/sbom/sbom_command.py b/cycode/cli/apps/report/sbom/sbom_command.py new file mode 100644 index 00000000..4454a966 --- /dev/null +++ b/cycode/cli/apps/report/sbom/sbom_command.py @@ -0,0 +1,69 @@ +from pathlib import Path +from typing import Annotated, Optional + +import click +import typer + +from cycode.cli.cli_types import SbomFormatOption, SbomOutputFormatOption +from cycode.cyclient.report_client import ReportParameters + +_OUTPUT_RICH_HELP_PANEL = 'Output options' + + +def sbom_command( + ctx: typer.Context, + sbom_format: Annotated[ + SbomFormatOption, + typer.Option( + '--format', + '-f', + help='SBOM format.', + case_sensitive=False, + show_default=False, + ), + ], + output_format: Annotated[ + SbomOutputFormatOption, + typer.Option( + '--output-format', + '-o', + help='Specify the output file format.', + rich_help_panel=_OUTPUT_RICH_HELP_PANEL, + ), + ] = SbomOutputFormatOption.JSON, + output_file: Annotated[ + Optional[Path], + typer.Option( + help='Output file.', + show_default='Autogenerated filename saved to the current directory', + dir_okay=False, + writable=True, + rich_help_panel=_OUTPUT_RICH_HELP_PANEL, + ), + ] = None, + include_vulnerabilities: Annotated[ + bool, typer.Option('--include-vulnerabilities', help='Include vulnerabilities.', show_default=False) + ] = False, + include_dev_dependencies: Annotated[ + bool, typer.Option('--include-dev-dependencies', help='Include dev dependencies.', show_default=False) + ] = False, +) -> int: + """Generate SBOM report.""" + sbom_format_parts = sbom_format.split('-') + if len(sbom_format_parts) != 2: + raise click.ClickException('Invalid SBOM format.') + + sbom_format, sbom_format_version = sbom_format_parts + + report_parameters = ReportParameters( + entity_type='SbomCli', + sbom_report_type=sbom_format, + sbom_version=sbom_format_version, + output_format=output_format, + include_vulnerabilities=include_vulnerabilities, + include_dev_dependencies=include_dev_dependencies, + ) + ctx.obj['report_parameters'] = report_parameters + ctx.obj['output_file'] = output_file + + return 1 diff --git a/cycode/cli/apps/report/sbom/sbom_report_file.py b/cycode/cli/apps/report/sbom/sbom_report_file.py new file mode 100644 index 00000000..f3178b44 --- /dev/null +++ b/cycode/cli/apps/report/sbom/sbom_report_file.py @@ -0,0 +1,51 @@ +import os +import pathlib +import re +from typing import Optional + +import typer + +from cycode.cli.console import console + + +class SbomReportFile: + def __init__(self, storage_path: str, output_format: str, output_file: Optional[pathlib.Path]) -> None: + if output_file is None: + output_file = pathlib.Path(storage_path) + + output_ext = f'.{output_format}' + if output_file.suffix != output_ext: + output_file = output_file.with_suffix(output_ext) + + self._file_path = output_file + + def is_exists(self) -> bool: + return self._file_path.exists() + + def _prompt_overwrite(self) -> bool: + return typer.confirm(f'File {self._file_path} already exists. Save with a different filename?', default=True) + + def _write(self, content: str) -> None: + with open(self._file_path, 'w', encoding='UTF-8') as f: + f.write(content) + + def _notify_about_saved_file(self) -> None: + console.print(f'Report saved to {self._file_path}') + + def _find_and_set_unique_filename(self) -> None: + attempt_no = 0 + while self.is_exists(): + attempt_no += 1 + + base, ext = os.path.splitext(self._file_path) + # Remove previous suffix + base = re.sub(r'-\d+$', '', base) + + self._file_path = pathlib.Path(f'{base}-{attempt_no}{ext}') + + def write(self, content: str) -> None: + if self.is_exists() and self._prompt_overwrite(): + self._find_and_set_unique_filename() + + self._write(content) + self._notify_about_saved_file() diff --git a/cycode/cli/apps/report_import/__init__.py b/cycode/cli/apps/report_import/__init__.py new file mode 100644 index 00000000..1fd2475b --- /dev/null +++ b/cycode/cli/apps/report_import/__init__.py @@ -0,0 +1,8 @@ +import typer + +from cycode.cli.apps.report_import.report_import_command import report_import_command +from cycode.cli.apps.report_import.sbom import sbom_command + +app = typer.Typer(name='import', no_args_is_help=True) +app.callback(short_help='Import report. You`ll need to specify which report type to import.')(report_import_command) +app.command(name='sbom', short_help='Import SBOM report from a local path.')(sbom_command) diff --git a/cycode/cli/apps/report_import/report_import_command.py b/cycode/cli/apps/report_import/report_import_command.py new file mode 100644 index 00000000..3e346bbe --- /dev/null +++ b/cycode/cli/apps/report_import/report_import_command.py @@ -0,0 +1,10 @@ +import typer + + +def report_import_command(ctx: typer.Context) -> int: + """:bar_chart: [bold cyan]Import security reports.[/] + + Example usage: + * `cycode import sbom`: Import SBOM report + """ + return 1 diff --git a/cycode/cli/apps/report_import/sbom/__init__.py b/cycode/cli/apps/report_import/sbom/__init__.py new file mode 100644 index 00000000..4d667031 --- /dev/null +++ b/cycode/cli/apps/report_import/sbom/__init__.py @@ -0,0 +1,6 @@ +import typer + +from cycode.cli.apps.report_import.sbom.sbom_command import sbom_command + +app = typer.Typer(name='sbom') +app.command(name='path', short_help='Import SBOM report from a local path.')(sbom_command) diff --git a/cycode/cli/apps/report_import/sbom/sbom_command.py b/cycode/cli/apps/report_import/sbom/sbom_command.py new file mode 100644 index 00000000..b6b5dfeb --- /dev/null +++ b/cycode/cli/apps/report_import/sbom/sbom_command.py @@ -0,0 +1,73 @@ +from pathlib import Path +from typing import Annotated, Optional + +import typer + +from cycode.cli.cli_types import BusinessImpactOption +from cycode.cli.exceptions.handle_report_sbom_errors import handle_report_exception +from cycode.cli.utils.get_api_client import get_import_sbom_cycode_client +from cycode.cyclient.import_sbom_client import ImportSbomParameters + + +def sbom_command( + ctx: typer.Context, + path: Annotated[ + Path, + typer.Argument( + exists=True, resolve_path=True, dir_okay=False, readable=True, help='Path to SBOM file.', show_default=False + ), + ], + sbom_name: Annotated[ + str, typer.Option('--name', '-n', help='SBOM Name.', case_sensitive=False, show_default=False) + ], + vendor: Annotated[ + str, typer.Option('--vendor', '-v', help='Vendor Name.', case_sensitive=False, show_default=False) + ], + labels: Annotated[ + Optional[list[str]], + typer.Option( + '--label', '-l', help='Label, can be specified multiple times.', case_sensitive=False, show_default=False + ), + ] = None, + owners: Annotated[ + Optional[list[str]], + typer.Option( + '--owner', + '-o', + help='Email address of a user in Cycode platform, can be specified multiple times.', + case_sensitive=True, + show_default=False, + ), + ] = None, + business_impact: Annotated[ + BusinessImpactOption, + typer.Option( + '--business-impact', + '-b', + help='Business Impact.', + case_sensitive=True, + show_default=True, + ), + ] = BusinessImpactOption.MEDIUM, +) -> None: + """Import SBOM.""" + client = get_import_sbom_cycode_client(ctx) + + import_parameters = ImportSbomParameters( + Name=sbom_name, + Vendor=vendor, + BusinessImpact=business_impact, + Labels=labels, + Owners=owners, + ) + + try: + if not path.exists(): + from errno import ENOENT + from os import strerror + + raise FileNotFoundError(ENOENT, strerror(ENOENT), path.absolute()) + + client.request_sbom_import_execution(import_parameters, path) + except Exception as e: + handle_report_exception(ctx, e) diff --git a/cycode/cli/apps/sca_options.py b/cycode/cli/apps/sca_options.py new file mode 100644 index 00000000..3c904ee6 --- /dev/null +++ b/cycode/cli/apps/sca_options.py @@ -0,0 +1,47 @@ +from pathlib import Path +from typing import Annotated, Optional + +import typer + +_SCA_RICH_HELP_PANEL = 'SCA options' + +NoRestoreOption = Annotated[ + bool, + typer.Option( + '--no-restore', + help='When specified, Cycode will not run restore command. Will scan direct dependencies [b]only[/]!', + rich_help_panel=_SCA_RICH_HELP_PANEL, + ), +] + +GradleAllSubProjectsOption = Annotated[ + bool, + typer.Option( + '--gradle-all-sub-projects', + help='When specified, Cycode will run gradle restore command for all sub projects. ' + 'Should run from root project directory [b]only[/]!', + rich_help_panel=_SCA_RICH_HELP_PANEL, + ), +] + +MavenSettingsFileOption = Annotated[ + Optional[Path], + typer.Option( + '--maven-settings-file', + show_default=False, + help='When specified, Cycode will use this settings.xml file when building the maven dependency tree.', + dir_okay=False, + rich_help_panel=_SCA_RICH_HELP_PANEL, + ), +] + + +def apply_sca_restore_options_to_context( + ctx: typer.Context, + no_restore: bool, + gradle_all_sub_projects: bool, + maven_settings_file: Optional[Path], +) -> None: + ctx.obj['no_restore'] = no_restore + ctx.obj['gradle_all_sub_projects'] = gradle_all_sub_projects + ctx.obj['maven_settings_file'] = maven_settings_file diff --git a/cycode/cli/apps/scan/__init__.py b/cycode/cli/apps/scan/__init__.py new file mode 100644 index 00000000..629c3b8f --- /dev/null +++ b/cycode/cli/apps/scan/__init__.py @@ -0,0 +1,50 @@ +import typer + +from cycode.cli.apps.scan.commit_history.commit_history_command import commit_history_command +from cycode.cli.apps.scan.path.path_command import path_command +from cycode.cli.apps.scan.pre_commit.pre_commit_command import pre_commit_command +from cycode.cli.apps.scan.pre_push.pre_push_command import pre_push_command +from cycode.cli.apps.scan.pre_receive.pre_receive_command import pre_receive_command +from cycode.cli.apps.scan.repository.repository_command import repository_command +from cycode.cli.apps.scan.scan_command import scan_command, scan_command_result_callback + +app = typer.Typer(name='scan', no_args_is_help=True) + +_AUTOMATION_COMMANDS_RICH_HELP_PANEL = 'Automation commands' + +_scan_command_docs = 'https://github.com/cycodehq/cycode-cli/blob/main/README.md#scan-command' +_scan_command_epilog = f'[bold]Documentation:[/] [link={_scan_command_docs}]{_scan_command_docs}[/link]' + +app.callback( + short_help='Scan the content for Secrets, IaC, SCA, and SAST violations.', + result_callback=scan_command_result_callback, + epilog=_scan_command_epilog, +)(scan_command) + +app.command(name='path', short_help='Scan the files in the paths provided in the command.')(path_command) +app.command(name='repository', short_help='Scan the Git repository included files.')(repository_command) +app.command(name='commit-history', short_help='Scan commit history or perform diff scanning between specific commits.')( + commit_history_command +) +app.command( + name='pre-commit', + short_help='Use this command in pre-commit hook to scan any content that was not committed yet.', + rich_help_panel=_AUTOMATION_COMMANDS_RICH_HELP_PANEL, +)(pre_commit_command) +app.command( + name='pre-push', + short_help='Use this command in pre-push hook to scan commits before pushing them to the remote repository.', + rich_help_panel=_AUTOMATION_COMMANDS_RICH_HELP_PANEL, +)(pre_push_command) +app.command( + name='pre-receive', + short_help='Use this command in pre-receive hook ' + 'to scan commits on the server side before pushing them to the repository.', + rich_help_panel=_AUTOMATION_COMMANDS_RICH_HELP_PANEL, +)(pre_receive_command) + +# backward compatibility +app.command(hidden=True, name='commit_history')(commit_history_command) +app.command(hidden=True, name='pre_commit')(pre_commit_command) +app.command(hidden=True, name='pre_push')(pre_push_command) +app.command(hidden=True, name='pre_receive')(pre_receive_command) diff --git a/cycode/cli/apps/scan/aggregation_report.py b/cycode/cli/apps/scan/aggregation_report.py new file mode 100644 index 00000000..45b891ed --- /dev/null +++ b/cycode/cli/apps/scan/aggregation_report.py @@ -0,0 +1,42 @@ +from typing import TYPE_CHECKING, Optional + +import typer + +from cycode.logger import get_logger + +if TYPE_CHECKING: + from cycode.cyclient.scan_client import ScanClient + +logger = get_logger('Aggregation Report URL') + + +def _set_aggregation_report_url(ctx: typer.Context, aggregation_report_url: Optional[str] = None) -> None: + ctx.obj['aggregation_report_url'] = aggregation_report_url + + +def try_get_aggregation_report_url_if_needed( + scan_parameters: dict, cycode_client: 'ScanClient', scan_type: str +) -> Optional[str]: + if not scan_parameters.get('report', False): + return None + + aggregation_id = scan_parameters.get('aggregation_id') + if aggregation_id is None: + return None + + try: + report_url_response = cycode_client.get_scan_aggregation_report_url(aggregation_id, scan_type) + return report_url_response.report_url + except Exception as e: + logger.debug('Failed to get aggregation report url: %s', str(e)) + + +def try_set_aggregation_report_url_if_needed( + ctx: typer.Context, scan_parameters: dict, cycode_client: 'ScanClient', scan_type: str +) -> None: + aggregation_report_url = try_get_aggregation_report_url_if_needed(scan_parameters, cycode_client, scan_type) + if aggregation_report_url: + _set_aggregation_report_url(ctx, aggregation_report_url) + logger.debug('Aggregation report URL set successfully', {'aggregation_report_url': aggregation_report_url}) + else: + logger.debug('No aggregation report URL found or report generation is disabled') diff --git a/cycode/cli/apps/scan/code_scanner.py b/cycode/cli/apps/scan/code_scanner.py new file mode 100644 index 00000000..616f22b3 --- /dev/null +++ b/cycode/cli/apps/scan/code_scanner.py @@ -0,0 +1,409 @@ +import os +import time +from platform import platform +from typing import TYPE_CHECKING, Callable, Optional + +import requests +import typer + +from cycode.cli import consts +from cycode.cli.apps.scan.aggregation_report import try_set_aggregation_report_url_if_needed +from cycode.cli.apps.scan.scan_parameters import get_scan_parameters +from cycode.cli.apps.scan.scan_result import ( + create_local_scan_result, + enrich_scan_result_with_data_from_detection_rules, + get_scan_result, + get_sync_scan_result, + print_local_scan_results, +) +from cycode.cli.config import configuration_manager +from cycode.cli.exceptions import custom_exceptions +from cycode.cli.exceptions.handle_scan_errors import handle_scan_exception +from cycode.cli.files_collector.path_documents import get_relevant_documents +from cycode.cli.files_collector.sca.sca_file_collector import add_sca_dependencies_tree_documents_if_needed +from cycode.cli.files_collector.zip_documents import zip_documents +from cycode.cli.models import CliError, Document, LocalScanResult +from cycode.cli.utils.path_utils import get_absolute_path, get_path_by_os +from cycode.cli.utils.progress_bar import ScanProgressBarSection +from cycode.cli.utils.scan_batch import run_parallel_batched_scan +from cycode.cli.utils.scan_utils import ( + generate_unique_scan_id, + is_cycodeignore_allowed_by_scan_config, + set_issue_detected_by_scan_results, + should_use_presigned_upload, +) +from cycode.cyclient.models import ZippedFileScanResult +from cycode.logger import get_logger + +if TYPE_CHECKING: + from cycode.cli.files_collector.models.in_memory_zip import InMemoryZip + from cycode.cli.printers.console_printer import ConsolePrinter + from cycode.cli.utils.progress_bar import BaseProgressBar + from cycode.cyclient.scan_client import ScanClient + +start_scan_time = time.time() + + +logger = get_logger('Code Scanner') + + +def scan_disk_files(ctx: typer.Context, paths: tuple[str, ...]) -> None: + scan_type = ctx.obj['scan_type'] + progress_bar = ctx.obj['progress_bar'] + + try: + documents = get_relevant_documents( + progress_bar, + ScanProgressBarSection.PREPARE_LOCAL_FILES, + scan_type, + paths, + is_cycodeignore_allowed=is_cycodeignore_allowed_by_scan_config(ctx), + ) + + # Add entrypoint.cycode file at root path to mark the scan root (only for single path that is a directory) + if len(paths) == 1: + root_path = paths[0] + absolute_root_path = get_absolute_path(root_path) + if os.path.isdir(absolute_root_path): + entrypoint_path = get_path_by_os(os.path.join(absolute_root_path, consts.CYCODE_ENTRYPOINT_FILENAME)) + entrypoint_document = Document( + entrypoint_path, + '', # Empty file content + is_git_diff_format=False, + absolute_path=entrypoint_path, + ) + documents.append(entrypoint_document) + + add_sca_dependencies_tree_documents_if_needed(ctx, scan_type, documents) + scan_documents(ctx, documents, get_scan_parameters(ctx, paths)) + except Exception as e: + handle_scan_exception(ctx, e) + + +def _should_use_sync_flow(command_scan_type: str, scan_type: str, sync_option: bool) -> bool: + """Decide whether to use sync flow or async flow for the scan. + + Note: + Passing `--sync` option does not mean that sync flow will be used in all cases. + + The logic: + - for IAC scan, sync flow is always used + - for SAST scan, sync flow is not supported + - for SCA and Secrets scan, sync flow is supported only for path/repository scan + + """ + if not sync_option and scan_type != consts.IAC_SCAN_TYPE: + return False + + if command_scan_type not in {'path', 'repository', 'ai_guardrails'}: + return False + + if scan_type == consts.IAC_SCAN_TYPE: + # sync in the only available flow for IAC scan; we do not use detector directly anymore + return True + + if scan_type is consts.SAST_SCAN_TYPE: # noqa: SIM103 + # SAST does not support sync flow + return False + + return True + + +def _get_scan_documents_thread_func( + ctx: typer.Context, + is_git_diff: bool, + is_commit_range: bool, + scan_parameters: dict, +) -> Callable[[list[Document]], tuple[str, CliError, LocalScanResult]]: + cycode_client = ctx.obj['client'] + scan_type = ctx.obj['scan_type'] + severity_threshold = ctx.obj['severity_threshold'] + sync_option = ctx.obj['sync'] + command_scan_type = ctx.info_name + + def _scan_batch_thread_func(batch: list[Document]) -> tuple[str, CliError, LocalScanResult]: + local_scan_result = error = error_message = None + detections_count = relevant_detections_count = zip_file_size = 0 + + scan_id = str(generate_unique_scan_id()) + scan_completed = False + + should_use_sync_flow = _should_use_sync_flow(command_scan_type, scan_type, sync_option) + + try: + logger.debug('Preparing local files, %s', {'batch_files_count': len(batch)}) + zipped_documents = zip_documents(scan_type, batch) + zip_file_size = zipped_documents.size + scan_result = _perform_scan( + cycode_client, + zipped_documents, + scan_type, + is_git_diff, + is_commit_range, + scan_parameters, + should_use_sync_flow, + ) + + enrich_scan_result_with_data_from_detection_rules(cycode_client, scan_result) + + local_scan_result = create_local_scan_result( + scan_result, batch, command_scan_type, scan_type, severity_threshold + ) + + scan_completed = True + except Exception as e: + error = handle_scan_exception(ctx, e, return_exception=True) + error_message = str(e) + + if local_scan_result: + detections_count = local_scan_result.detections_count + relevant_detections_count = local_scan_result.relevant_detections_count + scan_id = local_scan_result.scan_id + + logger.debug( + 'Processing scan results, %s', + { + 'all_violations_count': detections_count, + 'relevant_violations_count': relevant_detections_count, + 'scan_id': scan_id, + 'zip_file_size': zip_file_size, + }, + ) + report_scan_status( + cycode_client, + scan_type, + scan_id, + scan_completed, + relevant_detections_count, + detections_count, + len(batch), + zip_file_size, + command_scan_type, + error_message, + ) + + return scan_id, error, local_scan_result + + return _scan_batch_thread_func + + +def _run_presigned_upload_scan( + scan_batch_thread_func: Callable, + scan_type: str, + documents_to_scan: list[Document], + progress_bar: 'BaseProgressBar', + printer: 'ConsolePrinter', +) -> tuple: + try: + # Try to zip all documents as a single batch; ZipTooLargeError raised if it exceeds the scan type's limit + zip_documents(scan_type, documents_to_scan) + # It fits: skip batching and upload everything as one ZIP + return run_parallel_batched_scan( + scan_batch_thread_func, + scan_type, + documents_to_scan, + progress_bar=progress_bar, + skip_batching=True, + ) + except custom_exceptions.ZipTooLargeError: + printer.print_warning( + 'The scan is too large to upload as a single file. This may result in corrupted scan results.' + ) + return run_parallel_batched_scan( + scan_batch_thread_func, + scan_type, + documents_to_scan, + progress_bar=progress_bar, + ) + + +def scan_documents( + ctx: typer.Context, + documents_to_scan: list[Document], + scan_parameters: dict, + is_git_diff: bool = False, + is_commit_range: bool = False, +) -> None: + scan_type = ctx.obj['scan_type'] + progress_bar = ctx.obj['progress_bar'] + printer = ctx.obj.get('console_printer') + + if not documents_to_scan: + progress_bar.stop() + printer.print_error( + CliError( + code='no_relevant_files', + message='Error: The scan could not be completed - relevant files to scan are not found. ' + 'Enable verbose mode to see more details.', + ) + ) + return + + scan_batch_thread_func = _get_scan_documents_thread_func(ctx, is_git_diff, is_commit_range, scan_parameters) + + if should_use_presigned_upload(scan_type): + errors, local_scan_results = _run_presigned_upload_scan( + scan_batch_thread_func, scan_type, documents_to_scan, progress_bar, printer + ) + else: + errors, local_scan_results = run_parallel_batched_scan( + scan_batch_thread_func, scan_type, documents_to_scan, progress_bar=progress_bar + ) + + try_set_aggregation_report_url_if_needed(ctx, scan_parameters, ctx.obj['client'], scan_type) + + progress_bar.set_section_length(ScanProgressBarSection.GENERATE_REPORT, 1) + progress_bar.update(ScanProgressBarSection.GENERATE_REPORT) + progress_bar.stop() + + set_issue_detected_by_scan_results(ctx, local_scan_results) + print_local_scan_results(ctx, local_scan_results, errors) + + +def _perform_scan_v4_async( + cycode_client: 'ScanClient', + zipped_documents: 'InMemoryZip', + scan_type: str, + scan_parameters: dict, + is_git_diff: bool, + is_commit_range: bool, +) -> ZippedFileScanResult: + upload_link = cycode_client.get_upload_link(scan_type) + logger.debug('Got upload link, %s', {'upload_id': upload_link.upload_id}) + + cycode_client.upload_to_presigned_post(upload_link.url, upload_link.presigned_post_fields, zipped_documents) + logger.debug('Uploaded zip to presigned URL') + + scan_async_result = cycode_client.scan_repository_from_upload_id( + scan_type, upload_link.upload_id, scan_parameters, is_git_diff, is_commit_range + ) + logger.debug( + 'Presigned upload scan request triggered, %s', + {'scan_id': scan_async_result.scan_id, 'upload_id': upload_link.upload_id}, + ) + + return poll_scan_results(cycode_client, scan_async_result.scan_id, scan_type, scan_parameters) + + +def _perform_scan_async( + cycode_client: 'ScanClient', + zipped_documents: 'InMemoryZip', + scan_type: str, + scan_parameters: dict, + is_commit_range: bool, +) -> ZippedFileScanResult: + scan_async_result = cycode_client.zipped_file_scan_async( + zipped_documents, scan_type, scan_parameters, is_commit_range=is_commit_range + ) + logger.debug('Async scan request has been triggered successfully, %s', {'scan_id': scan_async_result.scan_id}) + + return poll_scan_results( + cycode_client, + scan_async_result.scan_id, + scan_type, + scan_parameters, + ) + + +def _perform_scan_sync( + cycode_client: 'ScanClient', + zipped_documents: 'InMemoryZip', + scan_type: str, + scan_parameters: dict, + is_git_diff: bool = False, +) -> 'ZippedFileScanResult': + scan_results = cycode_client.zipped_file_scan_sync(zipped_documents, scan_type, scan_parameters, is_git_diff) + logger.debug('Sync scan request has been triggered successfully, %s', {'scan_id': scan_results.id}) + return get_sync_scan_result(scan_type, scan_results) + + +def _perform_scan( + cycode_client: 'ScanClient', + zipped_documents: 'InMemoryZip', + scan_type: str, + is_git_diff: bool, + is_commit_range: bool, + scan_parameters: dict, + should_use_sync_flow: bool = False, +) -> ZippedFileScanResult: + if should_use_sync_flow: + # it does not support commit range scans; should_use_sync_flow handles it + return _perform_scan_sync(cycode_client, zipped_documents, scan_type, scan_parameters, is_git_diff) + + if should_use_presigned_upload(scan_type): + try: + return _perform_scan_v4_async( + cycode_client, zipped_documents, scan_type, scan_parameters, is_git_diff, is_commit_range + ) + except requests.exceptions.RequestException: + logger.warning('Direct upload to object storage failed. Falling back to upload via Cycode API. ') + + return _perform_scan_async(cycode_client, zipped_documents, scan_type, scan_parameters, is_commit_range) + + +def poll_scan_results( + cycode_client: 'ScanClient', + scan_id: str, + scan_type: str, + scan_parameters: dict, + polling_timeout: Optional[int] = None, +) -> 'ZippedFileScanResult': + if polling_timeout is None: + polling_timeout = configuration_manager.get_scan_polling_timeout_in_seconds() + + last_scan_update_at = None + end_polling_time = time.time() + polling_timeout + + while time.time() < end_polling_time: + scan_details = cycode_client.get_scan_details(scan_type, scan_id) + + if scan_details.scan_update_at is not None and scan_details.scan_update_at != last_scan_update_at: + last_scan_update_at = scan_details.scan_update_at + logger.debug('Scan update, %s', {'scan_id': scan_details.id, 'scan_status': scan_details.scan_status}) + + if scan_details.message: + logger.debug('Scan message: %s', scan_details.message) + + if scan_details.scan_status == consts.SCAN_STATUS_COMPLETED: + return get_scan_result(cycode_client, scan_type, scan_id, scan_details, scan_parameters) + + if scan_details.scan_status == consts.SCAN_STATUS_ERROR: + raise custom_exceptions.ScanAsyncError( + f'Error occurred while trying to scan zip file. {scan_details.message}' + ) + + time.sleep(consts.SCAN_POLLING_WAIT_INTERVAL_IN_SECONDS) + + raise custom_exceptions.ScanAsyncError(f'Failed to complete scan after {polling_timeout} seconds') + + +def report_scan_status( + cycode_client: 'ScanClient', + scan_type: str, + scan_id: str, + scan_completed: bool, + output_detections_count: int, + all_detections_count: int, + files_to_scan_count: int, + zip_size: int, + command_scan_type: str, + error_message: Optional[str], +) -> None: + try: + end_scan_time = time.time() + scan_status = { + 'zip_size': zip_size, + 'execution_time': int(end_scan_time - start_scan_time), + 'output_detections_count': output_detections_count, + 'all_detections_count': all_detections_count, + 'scannable_files_count': files_to_scan_count, + 'status': 'Completed' if scan_completed else 'Error', + 'scan_command_type': command_scan_type, + 'operation_system': platform(), + 'error_message': error_message, + 'scan_type': scan_type, + } + + cycode_client.report_scan_status(scan_type, scan_id, scan_status) + except Exception as e: + logger.debug('Failed to report scan status', exc_info=e) diff --git a/cycode/cyclient/scan_config/__init__.py b/cycode/cli/apps/scan/commit_history/__init__.py similarity index 100% rename from cycode/cyclient/scan_config/__init__.py rename to cycode/cli/apps/scan/commit_history/__init__.py diff --git a/cycode/cli/apps/scan/commit_history/commit_history_command.py b/cycode/cli/apps/scan/commit_history/commit_history_command.py new file mode 100644 index 00000000..46d911e8 --- /dev/null +++ b/cycode/cli/apps/scan/commit_history/commit_history_command.py @@ -0,0 +1,30 @@ +from pathlib import Path +from typing import Annotated + +import typer + +from cycode.cli.apps.scan.commit_range_scanner import scan_commit_range +from cycode.cli.exceptions.handle_scan_errors import handle_scan_exception +from cycode.cli.logger import logger + + +def commit_history_command( + ctx: typer.Context, + path: Annotated[ + Path, typer.Argument(exists=True, resolve_path=True, help='Path to Git repository to scan', show_default=False) + ], + commit_range: Annotated[ + str, + typer.Option( + '--commit-range', + '-r', + help='Scan a commit range in this Git repository (example: HEAD~1)', + show_default='cycode scans all commit history', + ), + ] = '--all', +) -> None: + try: + logger.debug('Starting commit history scan process, %s', {'path': path, 'commit_range': commit_range}) + scan_commit_range(ctx, repo_path=str(path), commit_range=commit_range) + except Exception as e: + handle_scan_exception(ctx, e) diff --git a/cycode/cli/apps/scan/commit_range_scanner.py b/cycode/cli/apps/scan/commit_range_scanner.py new file mode 100644 index 00000000..d4ce4be8 --- /dev/null +++ b/cycode/cli/apps/scan/commit_range_scanner.py @@ -0,0 +1,418 @@ +import os +from typing import TYPE_CHECKING, Optional + +import click +import requests +import typer + +from cycode.cli import consts +from cycode.cli.apps.scan.code_scanner import ( + poll_scan_results, + report_scan_status, + scan_documents, +) +from cycode.cli.apps.scan.scan_parameters import get_scan_parameters +from cycode.cli.apps.scan.scan_result import ( + create_local_scan_result, + enrich_scan_result_with_data_from_detection_rules, + init_default_scan_result, + print_local_scan_results, +) +from cycode.cli.config import configuration_manager +from cycode.cli.exceptions.handle_scan_errors import handle_scan_exception +from cycode.cli.files_collector.commit_range_documents import ( + collect_commit_range_diff_documents, + get_commit_range_modified_documents, + get_diff_file_content, + get_diff_file_path, + get_pre_commit_modified_documents, + get_safe_head_reference_for_diff, + parse_commit_range, +) +from cycode.cli.files_collector.documents_walk_ignore import filter_documents_with_cycodeignore +from cycode.cli.files_collector.file_excluder import excluder +from cycode.cli.files_collector.models.in_memory_zip import InMemoryZip +from cycode.cli.files_collector.sca.sca_file_collector import ( + perform_sca_pre_commit_range_scan_actions, + perform_sca_pre_hook_range_scan_actions, +) +from cycode.cli.files_collector.zip_documents import zip_documents +from cycode.cli.models import Document +from cycode.cli.utils.git_proxy import git_proxy +from cycode.cli.utils.path_utils import get_path_by_os +from cycode.cli.utils.progress_bar import ScanProgressBarSection +from cycode.cli.utils.scan_utils import ( + generate_unique_scan_id, + is_cycodeignore_allowed_by_scan_config, + set_issue_detected_by_scan_results, + should_use_presigned_upload, +) +from cycode.cyclient.models import ZippedFileScanResult +from cycode.logger import get_logger + +if TYPE_CHECKING: + from cycode.cyclient.scan_client import ScanClient + +logger = get_logger('Commit Range Scanner') + + +def _does_git_push_option_have_value(value: str) -> bool: + option_count_env_value = os.getenv(consts.GIT_PUSH_OPTION_COUNT_ENV_VAR_NAME, '') + option_count = int(option_count_env_value) if option_count_env_value.isdigit() else 0 + return any(os.getenv(f'{consts.GIT_PUSH_OPTION_ENV_VAR_PREFIX}{i}') == value for i in range(option_count)) + + +def is_verbose_mode_requested_in_pre_receive_scan() -> bool: + return _does_git_push_option_have_value(consts.VERBOSE_SCAN_FLAG) + + +def should_skip_pre_receive_scan() -> bool: + return _does_git_push_option_have_value(consts.SKIP_SCAN_FLAG) + + +def _perform_commit_range_scan_async( + cycode_client: 'ScanClient', + from_commit_zipped_documents: 'InMemoryZip', + to_commit_zipped_documents: 'InMemoryZip', + scan_type: str, + scan_parameters: dict, + timeout: Optional[int] = None, +) -> ZippedFileScanResult: + scan_async_result = cycode_client.commit_range_scan_async( + from_commit_zipped_documents, to_commit_zipped_documents, scan_type, scan_parameters + ) + + logger.debug( + 'Async commit range scan request has been triggered successfully, %s', {'scan_id': scan_async_result.scan_id} + ) + return poll_scan_results(cycode_client, scan_async_result.scan_id, scan_type, scan_parameters, timeout) + + +def _perform_commit_range_scan_v4_async( + cycode_client: 'ScanClient', + from_commit_zipped_documents: 'InMemoryZip', + to_commit_zipped_documents: 'InMemoryZip', + scan_type: str, + scan_parameters: dict, + timeout: Optional[int] = None, +) -> ZippedFileScanResult: + from_upload_link = cycode_client.get_upload_link(scan_type) + logger.debug('Got from-commit upload link, %s', {'upload_id': from_upload_link.upload_id}) + + cycode_client.upload_to_presigned_post( + from_upload_link.url, from_upload_link.presigned_post_fields, from_commit_zipped_documents + ) + logger.debug('Uploaded from-commit zip') + + to_upload_link = cycode_client.get_upload_link(scan_type) + logger.debug('Got to-commit upload link, %s', {'upload_id': to_upload_link.upload_id}) + + cycode_client.upload_to_presigned_post( + to_upload_link.url, to_upload_link.presigned_post_fields, to_commit_zipped_documents + ) + logger.debug('Uploaded to-commit zip') + + scan_async_result = cycode_client.commit_range_scan_from_upload_ids( + scan_type, from_upload_link.upload_id, to_upload_link.upload_id, scan_parameters + ) + logger.debug('V4 commit range scan request triggered, %s', {'scan_id': scan_async_result.scan_id}) + + return poll_scan_results(cycode_client, scan_async_result.scan_id, scan_type, scan_parameters, timeout) + + +def _scan_commit_range_documents( + ctx: typer.Context, + from_documents_to_scan: list[Document], + to_documents_to_scan: list[Document], + scan_parameters: Optional[dict] = None, + timeout: Optional[int] = None, +) -> None: + cycode_client = ctx.obj['client'] + scan_type = ctx.obj['scan_type'] + severity_threshold = ctx.obj['severity_threshold'] + scan_command_type = ctx.info_name + progress_bar = ctx.obj['progress_bar'] + + local_scan_result = error_message = None + scan_completed = False + scan_id = str(generate_unique_scan_id()) + from_commit_zipped_documents = InMemoryZip() + to_commit_zipped_documents = InMemoryZip() + + try: + progress_bar.set_section_length(ScanProgressBarSection.SCAN, 1) + + scan_result = init_default_scan_result(scan_id) + if len(from_documents_to_scan) > 0 or len(to_documents_to_scan) > 0: + logger.debug('Preparing from-commit zip') + # for SAST it is files from to_commit with actual content to scan + from_commit_zipped_documents = zip_documents(scan_type, from_documents_to_scan) + + logger.debug('Preparing to-commit zip') + # for SAST it is files with diff between from_commit and to_commit + to_commit_zipped_documents = zip_documents(scan_type, to_documents_to_scan) + + if should_use_presigned_upload(scan_type): + try: + scan_result = _perform_commit_range_scan_v4_async( + cycode_client, + from_commit_zipped_documents, + to_commit_zipped_documents, + scan_type, + scan_parameters, + timeout, + ) + except requests.exceptions.RequestException: + logger.warning('Direct upload to object storage failed. Falling back to upload via Cycode API. ') + scan_result = _perform_commit_range_scan_async( + cycode_client, + from_commit_zipped_documents, + to_commit_zipped_documents, + scan_type, + scan_parameters, + timeout, + ) + else: + scan_result = _perform_commit_range_scan_async( + cycode_client, + from_commit_zipped_documents, + to_commit_zipped_documents, + scan_type, + scan_parameters, + timeout, + ) + enrich_scan_result_with_data_from_detection_rules(cycode_client, scan_result) + + progress_bar.update(ScanProgressBarSection.SCAN) + progress_bar.set_section_length(ScanProgressBarSection.GENERATE_REPORT, 1) + + documents_to_scan = to_documents_to_scan + if scan_type == consts.SAST_SCAN_TYPE: + # actually for SAST from_documents_to_scan is full files and to_documents_to_scan is diff files + documents_to_scan = from_documents_to_scan + + local_scan_result = create_local_scan_result( + scan_result, documents_to_scan, scan_command_type, scan_type, severity_threshold + ) + set_issue_detected_by_scan_results(ctx, [local_scan_result]) + + progress_bar.update(ScanProgressBarSection.GENERATE_REPORT) + progress_bar.stop() + + # errors will be handled with try-except block; printing will not occur on errors + print_local_scan_results(ctx, [local_scan_result]) + + scan_completed = True + except Exception as e: + handle_scan_exception(ctx, e) + error_message = str(e) + + zip_file_size = from_commit_zipped_documents.size + to_commit_zipped_documents.size + + detections_count = relevant_detections_count = 0 + if local_scan_result: + detections_count = local_scan_result.detections_count + relevant_detections_count = local_scan_result.relevant_detections_count + scan_id = local_scan_result.scan_id + + logger.debug( + 'Processing commit range scan results, %s', + { + 'all_violations_count': detections_count, + 'relevant_violations_count': relevant_detections_count, + 'scan_id': scan_id, + 'zip_file_size': zip_file_size, + }, + ) + report_scan_status( + cycode_client, + scan_type, + scan_id, + scan_completed, + relevant_detections_count, + detections_count, + len(to_documents_to_scan), + zip_file_size, + scan_command_type, + error_message, + ) + + +def _scan_sca_commit_range(ctx: typer.Context, repo_path: str, commit_range: str, **_) -> None: + scan_parameters = get_scan_parameters(ctx, (repo_path,)) + + from_commit_rev, to_commit_rev, _ = parse_commit_range(commit_range, repo_path) + from_commit_documents, to_commit_documents, _ = get_commit_range_modified_documents( + ctx.obj['progress_bar'], ScanProgressBarSection.PREPARE_LOCAL_FILES, repo_path, from_commit_rev, to_commit_rev + ) + from_commit_documents = excluder.exclude_irrelevant_documents_to_scan(consts.SCA_SCAN_TYPE, from_commit_documents) + to_commit_documents = excluder.exclude_irrelevant_documents_to_scan(consts.SCA_SCAN_TYPE, to_commit_documents) + + is_cycodeignore_allowed = is_cycodeignore_allowed_by_scan_config(ctx) + from_commit_documents = filter_documents_with_cycodeignore( + from_commit_documents, repo_path, is_cycodeignore_allowed + ) + to_commit_documents = filter_documents_with_cycodeignore(to_commit_documents, repo_path, is_cycodeignore_allowed) + + perform_sca_pre_commit_range_scan_actions( + repo_path, from_commit_documents, from_commit_rev, to_commit_documents, to_commit_rev + ) + + _scan_commit_range_documents(ctx, from_commit_documents, to_commit_documents, scan_parameters=scan_parameters) + + +def _scan_secret_commit_range( + ctx: typer.Context, repo_path: str, commit_range: str, max_commits_count: Optional[int] = None +) -> None: + commit_diff_documents_to_scan = collect_commit_range_diff_documents(ctx, repo_path, commit_range, max_commits_count) + diff_documents_to_scan = excluder.exclude_irrelevant_documents_to_scan( + consts.SECRET_SCAN_TYPE, commit_diff_documents_to_scan + ) + + is_cycodeignore_allowed = is_cycodeignore_allowed_by_scan_config(ctx) + diff_documents_to_scan = filter_documents_with_cycodeignore( + diff_documents_to_scan, repo_path, is_cycodeignore_allowed + ) + + scan_documents( + ctx, diff_documents_to_scan, get_scan_parameters(ctx, (repo_path,)), is_git_diff=True, is_commit_range=True + ) + + +def _scan_sast_commit_range(ctx: typer.Context, repo_path: str, commit_range: str, **_) -> None: + scan_parameters = get_scan_parameters(ctx, (repo_path,)) + + from_commit_rev, to_commit_rev, _ = parse_commit_range(commit_range, repo_path) + _, commit_documents, diff_documents = get_commit_range_modified_documents( + ctx.obj['progress_bar'], + ScanProgressBarSection.PREPARE_LOCAL_FILES, + repo_path, + from_commit_rev, + to_commit_rev, + reverse_diff=False, + ) + + commit_documents = excluder.exclude_irrelevant_documents_to_scan(consts.SAST_SCAN_TYPE, commit_documents) + diff_documents = excluder.exclude_irrelevant_documents_to_scan(consts.SAST_SCAN_TYPE, diff_documents) + + is_cycodeignore_allowed = is_cycodeignore_allowed_by_scan_config(ctx) + commit_documents = filter_documents_with_cycodeignore(commit_documents, repo_path, is_cycodeignore_allowed) + diff_documents = filter_documents_with_cycodeignore(diff_documents, repo_path, is_cycodeignore_allowed) + + _scan_commit_range_documents(ctx, commit_documents, diff_documents, scan_parameters=scan_parameters) + + +_SCAN_TYPE_TO_COMMIT_RANGE_HANDLER = { + consts.SCA_SCAN_TYPE: _scan_sca_commit_range, + consts.SECRET_SCAN_TYPE: _scan_secret_commit_range, + consts.SAST_SCAN_TYPE: _scan_sast_commit_range, +} + + +def scan_commit_range(ctx: typer.Context, repo_path: str, commit_range: str, **kwargs) -> None: + scan_type = ctx.obj['scan_type'] + + progress_bar = ctx.obj['progress_bar'] + progress_bar.start() + + if scan_type not in _SCAN_TYPE_TO_COMMIT_RANGE_HANDLER: + raise click.ClickException(f'Commit range scanning for {scan_type.upper()} is not supported') + + _SCAN_TYPE_TO_COMMIT_RANGE_HANDLER[scan_type](ctx, repo_path, commit_range, **kwargs) + + +def _scan_sca_pre_commit(ctx: typer.Context, repo_path: str) -> None: + scan_parameters = get_scan_parameters(ctx) + + git_head_documents, pre_committed_documents, _ = get_pre_commit_modified_documents( + progress_bar=ctx.obj['progress_bar'], + progress_bar_section=ScanProgressBarSection.PREPARE_LOCAL_FILES, + repo_path=repo_path, + ) + + git_head_documents = excluder.exclude_irrelevant_documents_to_scan(consts.SCA_SCAN_TYPE, git_head_documents) + pre_committed_documents = excluder.exclude_irrelevant_documents_to_scan( + consts.SCA_SCAN_TYPE, pre_committed_documents + ) + + is_cycodeignore_allowed = is_cycodeignore_allowed_by_scan_config(ctx) + git_head_documents = filter_documents_with_cycodeignore(git_head_documents, repo_path, is_cycodeignore_allowed) + pre_committed_documents = filter_documents_with_cycodeignore( + pre_committed_documents, repo_path, is_cycodeignore_allowed + ) + + perform_sca_pre_hook_range_scan_actions(repo_path, git_head_documents, pre_committed_documents) + + _scan_commit_range_documents( + ctx, + git_head_documents, + pre_committed_documents, + scan_parameters, + configuration_manager.get_sca_pre_commit_timeout_in_seconds(), + ) + + +def _scan_secret_pre_commit(ctx: typer.Context, repo_path: str) -> None: + progress_bar = ctx.obj['progress_bar'] + repo = git_proxy.get_repo(repo_path) + head_reference = get_safe_head_reference_for_diff(repo) + diff_index = repo.index.diff(head_reference, create_patch=True, R=True) + + progress_bar.set_section_length(ScanProgressBarSection.PREPARE_LOCAL_FILES, len(diff_index)) + + documents_to_scan = [] + for diff in diff_index: + progress_bar.update(ScanProgressBarSection.PREPARE_LOCAL_FILES) + documents_to_scan.append( + Document( + get_path_by_os(get_diff_file_path(diff, repo=repo)), + get_diff_file_content(diff), + is_git_diff_format=True, + ) + ) + + documents_to_scan = excluder.exclude_irrelevant_documents_to_scan(consts.SECRET_SCAN_TYPE, documents_to_scan) + + is_cycodeignore_allowed = is_cycodeignore_allowed_by_scan_config(ctx) + documents_to_scan = filter_documents_with_cycodeignore(documents_to_scan, repo_path, is_cycodeignore_allowed) + + scan_documents(ctx, documents_to_scan, get_scan_parameters(ctx), is_git_diff=True) + + +def _scan_sast_pre_commit(ctx: typer.Context, repo_path: str, **_) -> None: + scan_parameters = get_scan_parameters(ctx, (repo_path,)) + + _, pre_committed_documents, diff_documents = get_pre_commit_modified_documents( + progress_bar=ctx.obj['progress_bar'], + progress_bar_section=ScanProgressBarSection.PREPARE_LOCAL_FILES, + repo_path=repo_path, + ) + + pre_committed_documents = excluder.exclude_irrelevant_documents_to_scan( + consts.SAST_SCAN_TYPE, pre_committed_documents + ) + diff_documents = excluder.exclude_irrelevant_documents_to_scan(consts.SAST_SCAN_TYPE, diff_documents) + + is_cycodeignore_allowed = is_cycodeignore_allowed_by_scan_config(ctx) + pre_committed_documents = filter_documents_with_cycodeignore( + pre_committed_documents, repo_path, is_cycodeignore_allowed + ) + diff_documents = filter_documents_with_cycodeignore(diff_documents, repo_path, is_cycodeignore_allowed) + + _scan_commit_range_documents(ctx, pre_committed_documents, diff_documents, scan_parameters=scan_parameters) + + +_SCAN_TYPE_TO_PRE_COMMIT_HANDLER = { + consts.SCA_SCAN_TYPE: _scan_sca_pre_commit, + consts.SECRET_SCAN_TYPE: _scan_secret_pre_commit, + consts.SAST_SCAN_TYPE: _scan_sast_pre_commit, +} + + +def scan_pre_commit(ctx: typer.Context, repo_path: str) -> None: + scan_type = ctx.obj['scan_type'] + if scan_type not in _SCAN_TYPE_TO_PRE_COMMIT_HANDLER: + raise click.ClickException(f'Pre-commit scanning for {scan_type.upper()} is not supported') + + _SCAN_TYPE_TO_PRE_COMMIT_HANDLER[scan_type](ctx, repo_path) + logger.debug('Pre-commit scan completed successfully') diff --git a/cycode/cli/apps/scan/detection_excluder.py b/cycode/cli/apps/scan/detection_excluder.py new file mode 100644 index 00000000..3697bcaf --- /dev/null +++ b/cycode/cli/apps/scan/detection_excluder.py @@ -0,0 +1,153 @@ +from typing import Optional + +from cycode.cli import consts +from cycode.cli.cli_types import SeverityOption +from cycode.cli.config import configuration_manager +from cycode.cli.models import DocumentDetections +from cycode.cyclient.models import Detection +from cycode.logger import get_logger + +logger = get_logger('Detection Excluder') + + +def _does_severity_match_severity_threshold(severity: str, severity_threshold: str) -> bool: + detection_severity_value = SeverityOption.get_member_weight(severity) + severity_threshold_value = SeverityOption.get_member_weight(severity_threshold) + if detection_severity_value < 0 or severity_threshold_value < 0: + return True + + return detection_severity_value >= severity_threshold_value + + +def _exclude_irrelevant_detections( + detections: list[Detection], scan_type: str, command_scan_type: str, severity_threshold: str +) -> list[Detection]: + relevant_detections = _exclude_detections_by_exclusions_configuration(detections, scan_type) + relevant_detections = _exclude_detections_by_scan_type(relevant_detections, scan_type, command_scan_type) + return _exclude_detections_by_severity(relevant_detections, severity_threshold) + + +def _exclude_detections_by_severity(detections: list[Detection], severity_threshold: str) -> list[Detection]: + relevant_detections = [] + for detection in detections: + severity = detection.severity + + if _does_severity_match_severity_threshold(severity, severity_threshold): + relevant_detections.append(detection) + else: + logger.debug( + 'Going to ignore violations because they are below the severity threshold, %s', + {'severity': severity, 'severity_threshold': severity_threshold}, + ) + + return relevant_detections + + +def _exclude_detections_by_scan_type( + detections: list[Detection], scan_type: str, command_scan_type: str +) -> list[Detection]: + if command_scan_type == consts.PRE_COMMIT_COMMAND_SCAN_TYPE: + return _exclude_detections_in_deleted_lines(detections) + + exclude_in_deleted_lines = configuration_manager.get_should_exclude_detections_in_deleted_lines(command_scan_type) + if ( + command_scan_type in consts.COMMIT_RANGE_BASED_COMMAND_SCAN_TYPES + and scan_type == consts.SECRET_SCAN_TYPE + and exclude_in_deleted_lines + ): + return _exclude_detections_in_deleted_lines(detections) + + return detections + + +def _exclude_detections_in_deleted_lines(detections: list[Detection]) -> list[Detection]: + return [detection for detection in detections if detection.detection_details.get('line_type') != 'Removed'] + + +def _exclude_detections_by_exclusions_configuration(detections: list[Detection], scan_type: str) -> list[Detection]: + exclusions = configuration_manager.get_exclusions_by_scan_type(scan_type) + return [detection for detection in detections if not _should_exclude_detection(detection, exclusions)] + + +def _should_exclude_detection(detection: Detection, exclusions: dict) -> bool: + # FIXME(MarshalX): what the difference between by_value and by_sha? + exclusions_by_value = exclusions.get(consts.EXCLUSIONS_BY_VALUE_SECTION_NAME, []) + if _is_detection_sha_configured_in_exclusions(detection, exclusions_by_value): + logger.debug( + 'Ignoring violation because its value is on the ignore list, %s', + {'value_sha': detection.detection_details.get('sha512')}, + ) + return True + + exclusions_by_sha = exclusions.get(consts.EXCLUSIONS_BY_SHA_SECTION_NAME, []) + if _is_detection_sha_configured_in_exclusions(detection, exclusions_by_sha): + logger.debug( + 'Ignoring violation because its SHA value is on the ignore list, %s', + {'sha': detection.detection_details.get('sha512')}, + ) + return True + + exclusions_by_rule = exclusions.get(consts.EXCLUSIONS_BY_RULE_SECTION_NAME, []) + detection_rule_id = detection.detection_rule_id + if detection_rule_id in exclusions_by_rule: + logger.debug( + 'Ignoring violation because its Detection Rule ID is on the ignore list, %s', + {'detection_rule_id': detection_rule_id}, + ) + return True + + exclusions_by_package = exclusions.get(consts.EXCLUSIONS_BY_PACKAGE_SECTION_NAME, []) + package = _get_package_name(detection) + if package and package in exclusions_by_package: + logger.debug('Ignoring violation because its package@version is on the ignore list, %s', {'package': package}) + return True + + exclusions_by_cve = exclusions.get(consts.EXCLUSIONS_BY_CVE_SECTION_NAME, []) + cve = _get_cve_identifier(detection) + if cve and cve in exclusions_by_cve: + logger.debug('Ignoring violation because its CVE is on the ignore list, %s', {'cve': cve}) + return True + + return False + + +def _is_detection_sha_configured_in_exclusions(detection: Detection, exclusions: list[str]) -> bool: + detection_sha = detection.detection_details.get('sha512') + return detection_sha in exclusions + + +def _get_package_name(detection: Detection) -> Optional[str]: + package_name = detection.detection_details.get('vulnerable_component') + package_version = detection.detection_details.get('vulnerable_component_version') + + if package_name is None: + package_name = detection.detection_details.get('package_name') + package_version = detection.detection_details.get('package_version') + + if package_name and package_version: + return f'{package_name}@{package_version}' + + return None + + +def _get_cve_identifier(detection: Detection) -> Optional[str]: + return detection.detection_details.get('alert', {}).get('cve_identifier') + + +def exclude_irrelevant_document_detections( + document_detections_list: list[DocumentDetections], + scan_type: str, + command_scan_type: str, + severity_threshold: str, +) -> list[DocumentDetections]: + relevant_document_detections_list = [] + for document_detections in document_detections_list: + relevant_detections = _exclude_irrelevant_detections( + document_detections.detections, scan_type, command_scan_type, severity_threshold + ) + if relevant_detections: + relevant_document_detections_list.append( + DocumentDetections(document=document_detections.document, detections=relevant_detections) + ) + + return relevant_document_detections_list diff --git a/cycode/cli/apps/scan/path/__init__.py b/cycode/cli/apps/scan/path/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cycode/cli/apps/scan/path/path_command.py b/cycode/cli/apps/scan/path/path_command.py new file mode 100644 index 00000000..6b2beab5 --- /dev/null +++ b/cycode/cli/apps/scan/path/path_command.py @@ -0,0 +1,22 @@ +from pathlib import Path +from typing import Annotated + +import typer + +from cycode.cli.apps.scan.code_scanner import scan_disk_files +from cycode.cli.logger import logger + + +def path_command( + ctx: typer.Context, + paths: Annotated[ + list[Path], typer.Argument(exists=True, resolve_path=True, help='Paths to scan', show_default=False) + ], +) -> None: + progress_bar = ctx.obj['progress_bar'] + progress_bar.start() + + logger.debug('Starting path scan process, %s', {'paths': paths}) + + tuple_paths = tuple(str(path) for path in paths) + scan_disk_files(ctx, tuple_paths) diff --git a/cycode/cli/apps/scan/pre_commit/__init__.py b/cycode/cli/apps/scan/pre_commit/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cycode/cli/apps/scan/pre_commit/pre_commit_command.py b/cycode/cli/apps/scan/pre_commit/pre_commit_command.py new file mode 100644 index 00000000..e0cbc7a8 --- /dev/null +++ b/cycode/cli/apps/scan/pre_commit/pre_commit_command.py @@ -0,0 +1,18 @@ +import os +from typing import Annotated, Optional + +import typer + +from cycode.cli.apps.scan.commit_range_scanner import scan_pre_commit + + +def pre_commit_command( + ctx: typer.Context, + _: Annotated[Optional[list[str]], typer.Argument(help='Ignored arguments', hidden=True)] = None, +) -> None: + repo_path = os.getcwd() # change locally for easy testing + + progress_bar = ctx.obj['progress_bar'] + progress_bar.start() + + scan_pre_commit(ctx, repo_path) diff --git a/cycode/cli/apps/scan/pre_push/__init__.py b/cycode/cli/apps/scan/pre_push/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cycode/cli/apps/scan/pre_push/pre_push_command.py b/cycode/cli/apps/scan/pre_push/pre_push_command.py new file mode 100644 index 00000000..d3339ea9 --- /dev/null +++ b/cycode/cli/apps/scan/pre_push/pre_push_command.py @@ -0,0 +1,65 @@ +import logging +import os +from typing import Annotated, Optional + +import typer + +from cycode.cli import consts +from cycode.cli.apps.scan.commit_range_scanner import ( + is_verbose_mode_requested_in_pre_receive_scan, + scan_commit_range, + should_skip_pre_receive_scan, +) +from cycode.cli.config import configuration_manager +from cycode.cli.console import console +from cycode.cli.exceptions.handle_scan_errors import handle_scan_exception +from cycode.cli.files_collector.commit_range_documents import ( + calculate_pre_push_commit_range, + parse_pre_push_input, +) +from cycode.cli.logger import logger +from cycode.cli.utils import scan_utils +from cycode.cli.utils.task_timer import TimeoutAfter +from cycode.logger import set_logging_level + + +def pre_push_command( + ctx: typer.Context, + _: Annotated[Optional[list[str]], typer.Argument(help='Ignored arguments', hidden=True)] = None, +) -> None: + try: + if should_skip_pre_receive_scan(): + logger.info( + 'A scan has been skipped as per your request. ' + 'Please note that this may leave your system vulnerable to secrets that have not been detected.' + ) + return + + if is_verbose_mode_requested_in_pre_receive_scan(): + ctx.obj['verbose'] = True + set_logging_level(logging.DEBUG) + logger.debug('Verbose mode enabled: all log levels will be displayed.') + + command_scan_type = ctx.info_name + timeout = configuration_manager.get_pre_push_command_timeout(command_scan_type) + with TimeoutAfter(timeout): + push_update_details = parse_pre_push_input() + commit_range = calculate_pre_push_commit_range(push_update_details) + if not commit_range: + logger.info( + 'No new commits found for pushed branch, %s', + {'push_update_details': push_update_details}, + ) + return + + scan_commit_range( + ctx=ctx, + repo_path=os.getcwd(), + commit_range=commit_range, + max_commits_count=configuration_manager.get_pre_push_max_commits_to_scan_count(command_scan_type), + ) + + if scan_utils.is_scan_failed(ctx): + console.print(consts.PRE_RECEIVE_AND_PUSH_REMEDIATION_MESSAGE) + except Exception as e: + handle_scan_exception(ctx, e) diff --git a/cycode/cli/apps/scan/pre_receive/__init__.py b/cycode/cli/apps/scan/pre_receive/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cycode/cli/apps/scan/pre_receive/pre_receive_command.py b/cycode/cli/apps/scan/pre_receive/pre_receive_command.py new file mode 100644 index 00000000..70abd4aa --- /dev/null +++ b/cycode/cli/apps/scan/pre_receive/pre_receive_command.py @@ -0,0 +1,67 @@ +import logging +import os +from typing import Annotated, Optional + +import typer + +from cycode.cli import consts +from cycode.cli.apps.scan.commit_range_scanner import ( + is_verbose_mode_requested_in_pre_receive_scan, + scan_commit_range, + should_skip_pre_receive_scan, +) +from cycode.cli.config import configuration_manager +from cycode.cli.console import console +from cycode.cli.exceptions.handle_scan_errors import handle_scan_exception +from cycode.cli.files_collector.commit_range_documents import ( + calculate_pre_receive_commit_range, + parse_pre_receive_input, +) +from cycode.cli.logger import logger +from cycode.cli.utils import scan_utils +from cycode.cli.utils.task_timer import TimeoutAfter +from cycode.logger import set_logging_level + + +def pre_receive_command( + ctx: typer.Context, + _: Annotated[Optional[list[str]], typer.Argument(help='Ignored arguments', hidden=True)] = None, +) -> None: + try: + if should_skip_pre_receive_scan(): + logger.info( + 'A scan has been skipped as per your request. ' + 'Please note that this may leave your system vulnerable to secrets that have not been detected.' + ) + return + + if is_verbose_mode_requested_in_pre_receive_scan(): + ctx.obj['verbose'] = True + set_logging_level(logging.DEBUG) + logger.debug('Verbose mode enabled: all log levels will be displayed.') + + command_scan_type = ctx.info_name + timeout = configuration_manager.get_pre_receive_command_timeout(command_scan_type) + with TimeoutAfter(timeout): + branch_update_details = parse_pre_receive_input() + commit_range = calculate_pre_receive_commit_range( + repo_path=os.getcwd(), branch_update_details=branch_update_details + ) + if not commit_range: + logger.info( + 'No new commits found for pushed branch, %s', + {'branch_update_details': branch_update_details}, + ) + return + + scan_commit_range( + ctx=ctx, + repo_path=os.getcwd(), + commit_range=commit_range, + max_commits_count=configuration_manager.get_pre_receive_max_commits_to_scan_count(command_scan_type), + ) + + if scan_utils.is_scan_failed(ctx): + console.print(consts.PRE_RECEIVE_AND_PUSH_REMEDIATION_MESSAGE) + except Exception as e: + handle_scan_exception(ctx, e) diff --git a/cycode/cli/apps/scan/remote_url_resolver.py b/cycode/cli/apps/scan/remote_url_resolver.py new file mode 100644 index 00000000..870115e2 --- /dev/null +++ b/cycode/cli/apps/scan/remote_url_resolver.py @@ -0,0 +1,151 @@ +from typing import Optional + +from cycode.cli import consts +from cycode.cli.utils.git_proxy import git_proxy +from cycode.cli.utils.shell_executor import shell +from cycode.cli.utils.url_utils import sanitize_repository_url +from cycode.logger import get_logger + +logger = get_logger('Remote URL Resolver') + + +def _get_plastic_repository_name(path: str) -> Optional[str]: + """Get the name of the Plastic repository from the current working directory. + + The command to execute is: + cm status --header --machinereadable --fieldseparator=":::" + + Example of status header in machine-readable format: + STATUS:::0:::Project/RepoName:::OrgName@ServerInfo + """ + try: + command = [ + 'cm', + 'status', + '--header', + '--machinereadable', + f'--fieldseparator={consts.PLASTIC_VCS_DATA_SEPARATOR}', + ] + + status = shell( + command=command, timeout=consts.PLASTIC_VSC_CLI_TIMEOUT, working_directory=path, silent_exc_info=True + ) + if not status: + logger.debug('Failed to get Plastic repository name (command failed)') + return None + + status_parts = status.split(consts.PLASTIC_VCS_DATA_SEPARATOR) + if len(status_parts) < 2: + logger.debug('Failed to parse Plastic repository name (command returned unexpected format)') + return None + + return status_parts[2].strip() + except Exception: + logger.debug('Failed to get Plastic repository name. Probably not a Plastic repository') + return None + + +def _get_plastic_repository_list(working_dir: Optional[str] = None) -> dict[str, str]: + """Get the list of Plastic repositories and their GUIDs. + + The command to execute is: + cm repo list --format="{repname}:::{repguid}" + + Example line with data: + Project/RepoName:::tapo1zqt-wn99-4752-h61m-7d9k79d40r4v + + Each line represents an individual repository. + """ + repo_name_to_guid = {} + + try: + command = ['cm', 'repo', 'ls', f'--format={{repname}}{consts.PLASTIC_VCS_DATA_SEPARATOR}{{repguid}}'] + + status = shell( + command=command, timeout=consts.PLASTIC_VSC_CLI_TIMEOUT, working_directory=working_dir, silent_exc_info=True + ) + if not status: + logger.debug('Failed to get Plastic repository list (command failed)') + return repo_name_to_guid + + status_lines = status.splitlines() + for line in status_lines: + data_parts = line.split(consts.PLASTIC_VCS_DATA_SEPARATOR) + if len(data_parts) < 2: + logger.debug('Failed to parse Plastic repository list line (unexpected format), %s', {'line': line}) + continue + + repo_name, repo_guid = data_parts + repo_name_to_guid[repo_name.strip()] = repo_guid.strip() + + return repo_name_to_guid + except Exception as e: + logger.debug('Failed to get Plastic repository list', exc_info=e) + return repo_name_to_guid + + +def _try_to_get_plastic_remote_url(path: str) -> Optional[str]: + repository_name = _get_plastic_repository_name(path) + if not repository_name: + return None + + repository_map = _get_plastic_repository_list(path) + if repository_name not in repository_map: + logger.debug('Failed to get Plastic repository GUID (repository not found in the list)') + return None + + repository_guid = repository_map[repository_name] + return f'{consts.PLASTIC_VCS_REMOTE_URI_PREFIX}{repository_guid}' + + +def _try_get_git_remote_url(path: str) -> Optional[str]: + try: + repo = git_proxy.get_repo(path, search_parent_directories=True) + remote_url = repo.remotes[0].config_reader.get('url') + logger.debug('Found Git remote URL, %s', {'remote_url': remote_url, 'repo_path': repo.working_dir}) + # Sanitize URL to remove any embedded credentials/tokens before returning + sanitized_url = sanitize_repository_url(remote_url) + if sanitized_url != remote_url: + logger.debug('Sanitized repository URL to remove credentials') + return sanitized_url + except Exception as e: + logger.debug('Failed to get Git remote URL. Probably not a Git repository', exc_info=e) + return None + + +def _try_get_any_remote_url(path: str) -> Optional[str]: + remote_url = _try_get_git_remote_url(path) + if not remote_url: + remote_url = _try_to_get_plastic_remote_url(path) + + return remote_url + + +def get_remote_url_scan_parameter(paths: tuple[str, ...]) -> Optional[str]: + remote_urls = set() + for path in paths: + # FIXME(MarshalX): perf issue. This looping will produce: + # - len(paths) Git subprocess calls in the worst case + # - len(paths)*2 Plastic SCM subprocess calls + remote_url = _try_get_any_remote_url(path) + if remote_url: + # URLs are already sanitized in _try_get_git_remote_url, but sanitize again as safety measure + sanitized_url = sanitize_repository_url(remote_url) + remote_urls.add(sanitized_url) + + if len(remote_urls) == 1: + # we are resolving remote_url only if all paths belong to the same repo (identical remote URLs), + # otherwise, the behavior is undefined + remote_url = remote_urls.pop() + + logger.debug( + 'Single remote URL found. Scan will be associated with organization, %s', {'remote_url': remote_url} + ) + return remote_url + + logger.debug( + 'Multiple different remote URLs found. Scan will not be associated with organization, %s', + {'remote_urls': remote_urls}, + ) + + return None diff --git a/cycode/cli/apps/scan/repository/__init__.py b/cycode/cli/apps/scan/repository/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cycode/cli/apps/scan/repository/repository_command.py b/cycode/cli/apps/scan/repository/repository_command.py new file mode 100644 index 00000000..e32fec0d --- /dev/null +++ b/cycode/cli/apps/scan/repository/repository_command.py @@ -0,0 +1,74 @@ +from pathlib import Path +from typing import Annotated, Optional + +import click +import typer + +from cycode.cli import consts +from cycode.cli.apps.scan.code_scanner import scan_documents +from cycode.cli.apps.scan.scan_parameters import get_scan_parameters +from cycode.cli.exceptions.handle_scan_errors import handle_scan_exception +from cycode.cli.files_collector.documents_walk_ignore import filter_documents_with_cycodeignore +from cycode.cli.files_collector.file_excluder import excluder +from cycode.cli.files_collector.repository_documents import get_git_repository_tree_file_entries +from cycode.cli.files_collector.sca.sca_file_collector import add_sca_dependencies_tree_documents_if_needed +from cycode.cli.logger import logger +from cycode.cli.models import Document +from cycode.cli.utils.path_utils import get_path_by_os +from cycode.cli.utils.progress_bar import ScanProgressBarSection +from cycode.cli.utils.scan_utils import is_cycodeignore_allowed_by_scan_config + + +def repository_command( + ctx: typer.Context, + path: Annotated[ + Path, typer.Argument(exists=True, resolve_path=True, help='Path to Git repository to scan.', show_default=False) + ], + branch: Annotated[ + Optional[str], typer.Option('--branch', '-b', help='Branch to scan.', show_default='default branch') + ] = None, +) -> None: + try: + logger.debug('Starting repository scan process, %s', {'path': path, 'branch': branch}) + + scan_type = ctx.obj['scan_type'] + monitor = ctx.obj.get('monitor') + if monitor and scan_type != consts.SCA_SCAN_TYPE: + raise click.ClickException('Monitor flag is currently supported for SCA scan type only') + + progress_bar = ctx.obj['progress_bar'] + progress_bar.start() + + file_entries = list(get_git_repository_tree_file_entries(str(path), branch)) + progress_bar.set_section_length(ScanProgressBarSection.PREPARE_LOCAL_FILES, len(file_entries)) + + documents_to_scan = [] + for blob in file_entries: + # FIXME(MarshalX): probably file could be tree or submodule too. we expect blob only + progress_bar.update(ScanProgressBarSection.PREPARE_LOCAL_FILES) + + absolute_path = get_path_by_os(blob.abspath) + file_path = get_path_by_os(blob.path) if monitor else absolute_path + documents_to_scan.append( + Document( + file_path, + blob.data_stream.read().decode('UTF-8', errors='replace'), + absolute_path=absolute_path, + ) + ) + + documents_to_scan = excluder.exclude_irrelevant_documents_to_scan(scan_type, documents_to_scan) + + is_cycodeignore_allowed = is_cycodeignore_allowed_by_scan_config(ctx) + documents_to_scan = filter_documents_with_cycodeignore(documents_to_scan, str(path), is_cycodeignore_allowed) + + add_sca_dependencies_tree_documents_if_needed(ctx, scan_type, documents_to_scan) + + # Store branch in context so it can be included in scan parameters + if branch: + ctx.obj['branch'] = branch + + logger.debug('Found all relevant files for scanning %s', {'path': path, 'branch': branch}) + scan_documents(ctx, documents_to_scan, get_scan_parameters(ctx, (str(path),))) + except Exception as e: + handle_scan_exception(ctx, e) diff --git a/cycode/cli/apps/scan/scan_ci/__init__.py b/cycode/cli/apps/scan/scan_ci/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cycode/cli/apps/scan/scan_ci/ci_integrations.py b/cycode/cli/apps/scan/scan_ci/ci_integrations.py new file mode 100644 index 00000000..3cb617a9 --- /dev/null +++ b/cycode/cli/apps/scan/scan_ci/ci_integrations.py @@ -0,0 +1,62 @@ +import os + +import click + +from cycode.cli.console import console + + +def github_action_range() -> str: + before_sha = os.getenv('BEFORE_SHA') + push_base_sha = os.getenv('BASE_SHA') + pr_base_sha = os.getenv('PR_BASE_SHA') + default_branch = os.getenv('DEFAULT_BRANCH') + head_sha = os.getenv('GITHUB_SHA') + ref = os.getenv('GITHUB_REF') + + console.print(f'{before_sha}, {push_base_sha}, {pr_base_sha}, {default_branch}, {head_sha}, {ref}') + if before_sha and before_sha != NO_COMMITS: + return f'{before_sha}...' + + return '...' + + # if pr_base_sha and pr_base_sha != FIRST_COMMIT: + # + # if push_base_sha and push_base_sha != "null": + + +def circleci_range() -> str: + before_sha = os.getenv('BEFORE_SHA') + current_sha = os.getenv('CURRENT_SHA') + commit_range = f'{before_sha}...{current_sha}' + console.print(f'commit range: {commit_range}') + + if not commit_range.startswith('...'): + return commit_range + + commit_sha = os.getenv('CIRCLE_SHA1', 'HEAD') + + return f'{commit_sha}~1...' + + +def gitlab_range() -> str: + before_sha = os.getenv('CI_COMMIT_BEFORE_SHA') + commit_sha = os.getenv('CI_COMMIT_SHA', 'HEAD') + + if before_sha and before_sha != NO_COMMITS: + return f'{before_sha}...' + + return f'{commit_sha}' + + +def get_commit_range() -> str: + if os.getenv('GITHUB_ACTIONS'): + return github_action_range() + if os.getenv('CIRCLECI'): + return circleci_range() + if os.getenv('GITLAB_CI'): + return gitlab_range() + + raise click.ClickException('CI framework is not supported') + + +NO_COMMITS = '0000000000000000000000000000000000000000' diff --git a/cycode/cli/apps/scan/scan_ci/scan_ci_command.py b/cycode/cli/apps/scan/scan_ci/scan_ci_command.py new file mode 100644 index 00000000..7874a054 --- /dev/null +++ b/cycode/cli/apps/scan/scan_ci/scan_ci_command.py @@ -0,0 +1,18 @@ +import os + +import click +import typer + +from cycode.cli.apps.scan.commit_range_scanner import scan_commit_range +from cycode.cli.apps.scan.scan_ci.ci_integrations import get_commit_range + +# This command is not finished yet. It is not used in the codebase. + + +@click.command( + short_help='Execute scan in a CI environment which relies on the ' + 'CYCODE_TOKEN and CYCODE_REPO_LOCATION environment variables' +) +@click.pass_context +def scan_ci_command(ctx: typer.Context) -> None: + scan_commit_range(ctx, repo_path=os.getcwd(), commit_range=get_commit_range()) diff --git a/cycode/cli/apps/scan/scan_command.py b/cycode/cli/apps/scan/scan_command.py new file mode 100644 index 00000000..7aab9d27 --- /dev/null +++ b/cycode/cli/apps/scan/scan_command.py @@ -0,0 +1,179 @@ +import os +from pathlib import Path +from typing import Annotated, Optional + +import click +import typer + +from cycode.cli.apps.sca_options import ( + GradleAllSubProjectsOption, + MavenSettingsFileOption, + NoRestoreOption, + apply_sca_restore_options_to_context, +) +from cycode.cli.apps.scan.remote_url_resolver import _try_get_git_remote_url +from cycode.cli.cli_types import ExportTypeOption, ScanTypeOption, ScaScanTypeOption, SeverityOption +from cycode.cli.consts import ( + ISSUE_DETECTED_STATUS_CODE, + NO_ISSUES_STATUS_CODE, +) +from cycode.cli.files_collector.file_excluder import excluder +from cycode.cli.utils import scan_utils +from cycode.cli.utils.get_api_client import get_scan_cycode_client + +_EXPORT_RICH_HELP_PANEL = 'Export options' +_SCA_RICH_HELP_PANEL = 'SCA options' +_SECRET_RICH_HELP_PANEL = 'Secret options' + + +def scan_command( + ctx: typer.Context, + scan_type: Annotated[ + ScanTypeOption, + typer.Option( + '--scan-type', + '-t', + help='Specify the type of scan you wish to execute.', + case_sensitive=False, + ), + ] = ScanTypeOption.SECRET, + soft_fail: Annotated[ + bool, typer.Option('--soft-fail', help='Run the scan without failing; always return a non-error status code.') + ] = False, + severity_threshold: Annotated[ + SeverityOption, + typer.Option( + help='Show violations only for the specified level or higher.', + case_sensitive=False, + ), + ] = SeverityOption.INFO, + sync: Annotated[ + bool, + typer.Option( + '--sync', help='Run scan synchronously (INTERNAL FOR IDEs).', show_default='asynchronously', hidden=True + ), + ] = False, + report: Annotated[ + bool, + typer.Option( + '--cycode-report', + help='When specified, displays a link to the scan report in the Cycode platform in the console output.', + ), + ] = False, + show_secret: Annotated[ + bool, typer.Option('--show-secret', help='Show Secrets in plain text.', rich_help_panel=_SECRET_RICH_HELP_PANEL) + ] = False, + sca_scan: Annotated[ + list[ScaScanTypeOption], + typer.Option( + help='Specify the type of SCA scan you wish to execute.', + rich_help_panel=_SCA_RICH_HELP_PANEL, + ), + ] = (ScaScanTypeOption.PACKAGE_VULNERABILITIES, ScaScanTypeOption.LICENSE_COMPLIANCE), + monitor: Annotated[ + bool, + typer.Option( + '--monitor', + help='When specified, the scan results are recorded in the Discovery module.', + rich_help_panel=_SCA_RICH_HELP_PANEL, + ), + ] = False, + no_restore: NoRestoreOption = False, + gradle_all_sub_projects: GradleAllSubProjectsOption = False, + maven_settings_file: MavenSettingsFileOption = None, + export_type: Annotated[ + ExportTypeOption, + typer.Option( + '--export-type', + case_sensitive=False, + help='Specify the export type. ' + 'HTML and SVG will export terminal output and rely on --output option. ' + 'JSON always exports JSON.', + rich_help_panel=_EXPORT_RICH_HELP_PANEL, + ), + ] = None, + export_file: Annotated[ + Optional[Path], + typer.Option( + '--export-file', + help='Export file. Path to the file where the export will be saved.', + dir_okay=False, + writable=True, + rich_help_panel=_EXPORT_RICH_HELP_PANEL, + ), + ] = None, +) -> None: + """:mag: [bold cyan]Scan code for vulnerabilities (Secrets, IaC, SCA, SAST).[/] + + This command scans your code for various types of security issues, including: + * [yellow]Secrets:[/] Hardcoded credentials and sensitive information. + * [dodger_blue1]Infrastructure as Code (IaC):[/] Misconfigurations in Terraform, CloudFormation, etc. + * [green]Software Composition Analysis (SCA):[/] Vulnerabilities and license issues in dependencies. + * [magenta]Static Application Security Testing (SAST):[/] Code quality and security flaws. + + Example usage: + * `cycode scan path `: Scan a specific local directory or file. + * `cycode scan repository `: Scan Git related files in a local Git repository. + * `cycode scan commit-history `: Scan the commit history of a local Git repository. + + """ + if export_file and export_type is None: + raise typer.BadParameter( + 'Export type must be specified when --export-file is provided.', + param_hint='--export-type', + ) + if export_type and export_file is None: + raise typer.BadParameter( + 'Export file must be specified when --export-type is provided.', + param_hint='--export-file', + ) + + ctx.obj['show_secret'] = show_secret + ctx.obj['soft_fail'] = soft_fail + ctx.obj['scan_type'] = scan_type + ctx.obj['sync'] = sync + ctx.obj['severity_threshold'] = severity_threshold + ctx.obj['monitor'] = monitor + ctx.obj['report'] = report + apply_sca_restore_options_to_context(ctx, no_restore, gradle_all_sub_projects, maven_settings_file) + + scan_client = get_scan_cycode_client(ctx) + ctx.obj['client'] = scan_client + + # Get remote URL from current working directory + remote_url = _try_get_git_remote_url(os.getcwd()) + + remote_scan_config = scan_client.get_scan_configuration_safe(scan_type, remote_url) + if remote_scan_config: + excluder.apply_scan_config(str(scan_type), remote_scan_config) + + ctx.obj['scan_config'] = remote_scan_config + + if export_type and export_file: + console_printer = ctx.obj['console_printer'] + console_printer.enable_recording(export_type, export_file) + + _sca_scan_to_context(ctx, sca_scan) + + +def _sca_scan_to_context(ctx: typer.Context, sca_scan_user_selected: list[str]) -> None: + for sca_scan_option_selected in sca_scan_user_selected: + ctx.obj[sca_scan_option_selected] = True + + +@click.pass_context +def scan_command_result_callback(ctx: click.Context, *_, **__) -> None: + ctx.obj['scan_finalized'] = True + + progress_bar = ctx.obj.get('progress_bar') + if progress_bar: + progress_bar.stop() + + if ctx.obj['soft_fail']: + raise typer.Exit(0) + + exit_code = NO_ISSUES_STATUS_CODE + if scan_utils.is_scan_failed(ctx): + exit_code = ISSUE_DETECTED_STATUS_CODE + + raise typer.Exit(exit_code) diff --git a/cycode/cli/apps/scan/scan_parameters.py b/cycode/cli/apps/scan/scan_parameters.py new file mode 100644 index 00000000..58754e86 --- /dev/null +++ b/cycode/cli/apps/scan/scan_parameters.py @@ -0,0 +1,41 @@ +from typing import Optional + +import typer + +from cycode.cli.apps.scan.remote_url_resolver import get_remote_url_scan_parameter +from cycode.cli.utils.scan_utils import generate_unique_scan_id +from cycode.logger import get_logger + +logger = get_logger('Scan Parameters') + + +def _get_default_scan_parameters(ctx: typer.Context) -> dict: + return { + 'monitor': ctx.obj.get('monitor'), + 'report': ctx.obj.get('report'), + 'package_vulnerabilities': ctx.obj.get('package-vulnerabilities'), + 'license_compliance': ctx.obj.get('license-compliance'), + 'command_type': ctx.info_name.replace('-', '_'), # save backward compatibility + 'aggregation_id': str(generate_unique_scan_id()), + } + + +def get_scan_parameters(ctx: typer.Context, paths: Optional[tuple[str, ...]] = None) -> dict: + scan_parameters = _get_default_scan_parameters(ctx) + + if not paths: + return scan_parameters + + scan_parameters['paths'] = paths + + remote_url = get_remote_url_scan_parameter(paths) + # TODO(MarshalX): remove hardcode in context + ctx.obj['remote_url'] = remote_url + scan_parameters['remote_url'] = remote_url + + # Include branch information if available (for repository scans) + branch = ctx.obj.get('branch') + if branch: + scan_parameters['branch'] = branch + + return scan_parameters diff --git a/cycode/cli/apps/scan/scan_result.py b/cycode/cli/apps/scan/scan_result.py new file mode 100644 index 00000000..13fb8576 --- /dev/null +++ b/cycode/cli/apps/scan/scan_result.py @@ -0,0 +1,212 @@ +import os +from typing import TYPE_CHECKING, Optional + +import typer + +from cycode.cli import consts +from cycode.cli.apps.scan.aggregation_report import try_get_aggregation_report_url_if_needed +from cycode.cli.apps.scan.detection_excluder import exclude_irrelevant_document_detections +from cycode.cli.models import Document, DocumentDetections, LocalScanResult +from cycode.cli.utils.path_utils import get_path_by_os, normalize_file_path +from cycode.cyclient.models import ( + Detection, + DetectionSchema, + DetectionsPerFile, + ScanResultsSyncFlow, + ZippedFileScanResult, +) +from cycode.logger import get_logger + +if TYPE_CHECKING: + from cycode.cli.models import CliError + from cycode.cyclient.models import ScanDetailsResponse + from cycode.cyclient.scan_client import ScanClient + +logger = get_logger('Scan Results') + + +def _get_document_by_file_name( + documents: list[Document], file_name: str, unique_id: Optional[str] = None +) -> Optional[Document]: + for document in documents: + if normalize_file_path(document.path) == normalize_file_path(file_name) and document.unique_id == unique_id: + return document + + return None + + +def _get_document_detections( + scan_result: ZippedFileScanResult, documents_to_scan: list[Document] +) -> list[DocumentDetections]: + logger.debug('Getting document detections') + + document_detections = [] + for detections_per_file in scan_result.detections_per_file: + file_name = get_path_by_os(detections_per_file.file_name) + commit_id = detections_per_file.commit_id + + logger.debug( + 'Going to find the document of the violated file, %s', {'file_name': file_name, 'commit_id': commit_id} + ) + + document = _get_document_by_file_name(documents_to_scan, file_name, commit_id) + document_detections.append(DocumentDetections(document=document, detections=detections_per_file.detections)) + + return document_detections + + +def create_local_scan_result( + scan_result: ZippedFileScanResult, + documents_to_scan: list[Document], + command_scan_type: str, + scan_type: str, + severity_threshold: str, +) -> LocalScanResult: + document_detections = _get_document_detections(scan_result, documents_to_scan) + relevant_document_detections_list = exclude_irrelevant_document_detections( + document_detections, scan_type, command_scan_type, severity_threshold + ) + + detections_count = sum([len(document_detection.detections) for document_detection in document_detections]) + relevant_detections_count = sum( + [len(document_detections.detections) for document_detections in relevant_document_detections_list] + ) + + return LocalScanResult( + scan_id=scan_result.scan_id, + report_url=scan_result.report_url, + document_detections=relevant_document_detections_list, + issue_detected=len(relevant_document_detections_list) > 0, + detections_count=detections_count, + relevant_detections_count=relevant_detections_count, + ) + + +def _get_file_name_from_detection(scan_type: str, raw_detection: dict) -> str: + if scan_type == consts.SAST_SCAN_TYPE: + return raw_detection['detection_details']['file_path'] + if scan_type == consts.SECRET_SCAN_TYPE: + return _get_secret_file_name_from_detection(raw_detection) + + return raw_detection['detection_details']['file_path'] + + +def _get_secret_file_name_from_detection(raw_detection: dict) -> str: + file_path: str = raw_detection['detection_details']['file_path'] + file_name: str = raw_detection['detection_details']['file_name'] + return os.path.join(file_path, file_name) + + +def _map_detections_per_file_and_commit_id(scan_type: str, raw_detections: list[dict]) -> list[DetectionsPerFile]: + """Convert a list of detections (async flow) to list of DetectionsPerFile objects (sync flow). + + Args: + scan_type: Type of the scan. + raw_detections: List of detections as is returned from the server. + + Note: + This method fakes server response structure + to be able to use the same logic for both async and sync scans. + + Note: + Aggregation is performed by file name and commit ID (if available) + + """ + detections_per_files = {} + for raw_detection in raw_detections: + try: + # FIXME(MarshalX): investigate this field mapping + raw_detection['message'] = raw_detection['correlation_message'] + + file_name = _get_file_name_from_detection(scan_type, raw_detection) + detection: Detection = DetectionSchema().load(raw_detection) + commit_id: Optional[str] = detection.detection_details.get('commit_id') # could be None + group_by_key = (file_name, commit_id) + + if group_by_key in detections_per_files: + detections_per_files[group_by_key].append(detection) + else: + detections_per_files[group_by_key] = [detection] + except Exception as e: + logger.debug('Failed to parse detection', exc_info=e) + continue + + return [ + DetectionsPerFile(file_name=file_name, detections=file_detections, commit_id=commit_id) + for (file_name, commit_id), file_detections in detections_per_files.items() + ] + + +def init_default_scan_result(scan_id: str) -> ZippedFileScanResult: + return ZippedFileScanResult( + did_detect=False, + detections_per_file=[], + scan_id=scan_id, + ) + + +def get_scan_result( + cycode_client: 'ScanClient', + scan_type: str, + scan_id: str, + scan_details: 'ScanDetailsResponse', + scan_parameters: dict, +) -> ZippedFileScanResult: + if not scan_details.detections_count: + return init_default_scan_result(scan_id) + + scan_raw_detections = cycode_client.get_scan_raw_detections(scan_id) + + return ZippedFileScanResult( + did_detect=True, + detections_per_file=_map_detections_per_file_and_commit_id(scan_type, scan_raw_detections), + scan_id=scan_id, + report_url=try_get_aggregation_report_url_if_needed(scan_parameters, cycode_client, scan_type), + ) + + +def get_sync_scan_result(scan_type: str, scan_results: 'ScanResultsSyncFlow') -> ZippedFileScanResult: + return ZippedFileScanResult( + did_detect=True, + detections_per_file=_map_detections_per_file_and_commit_id(scan_type, scan_results.detection_messages), + scan_id=scan_results.id, + ) + + +def print_local_scan_results( + ctx: typer.Context, local_scan_results: list[LocalScanResult], errors: Optional[dict[str, 'CliError']] = None +) -> None: + printer = ctx.obj.get('console_printer') + printer.update_ctx(ctx) + printer.print_scan_results(local_scan_results, errors) + + +def enrich_scan_result_with_data_from_detection_rules( + cycode_client: 'ScanClient', scan_result: ZippedFileScanResult +) -> None: + detection_rule_ids = set() + for detections_per_file in scan_result.detections_per_file: + for detection in detections_per_file.detections: + detection_rule_ids.add(detection.detection_rule_id) + + detection_rules = cycode_client.get_detection_rules(detection_rule_ids) + detection_rules_by_id = {detection_rule.detection_rule_id: detection_rule for detection_rule in detection_rules} + + for detections_per_file in scan_result.detections_per_file: + for detection in detections_per_file.detections: + detection_rule = detection_rules_by_id.get(detection.detection_rule_id) + if not detection_rule: + # we want to make sure that BE returned it. better to not map data instead of failed scan + continue + + if not detection.severity and detection_rule.classification_data: + # it's fine to take the first one, because: + # - for "secrets" and "iac" there is only one classification rule per-detection rule + # - for "sca" and "sast" we get severity from detection service + detection.severity = detection_rule.classification_data[0].severity + + # detection_details never was typed properly. so not a problem for now + detection.detection_details['custom_remediation_guidelines'] = detection_rule.custom_remediation_guidelines + detection.detection_details['remediation_guidelines'] = detection_rule.remediation_guidelines + detection.detection_details['description'] = detection_rule.description + detection.detection_details['policy_display_name'] = detection_rule.display_name diff --git a/cycode/cli/apps/status/__init__.py b/cycode/cli/apps/status/__init__.py new file mode 100644 index 00000000..1161b2e6 --- /dev/null +++ b/cycode/cli/apps/status/__init__.py @@ -0,0 +1,8 @@ +import typer + +from cycode.cli.apps.status.status_command import status_command +from cycode.cli.apps.status.version_command import version_command + +app = typer.Typer(no_args_is_help=True) +app.command(name='status', short_help='Show the CLI status and exit.')(status_command) +app.command(name='version', hidden=True, short_help='Alias to status command.')(version_command) diff --git a/cycode/cli/apps/status/get_cli_status.py b/cycode/cli/apps/status/get_cli_status.py new file mode 100644 index 00000000..0cf6e8fd --- /dev/null +++ b/cycode/cli/apps/status/get_cli_status.py @@ -0,0 +1,49 @@ +import platform +from typing import TYPE_CHECKING + +from cycode import __version__ +from cycode.cli.apps.auth.auth_common import get_authorization_info +from cycode.cli.apps.status.models import CliStatus, CliSupportedModulesStatus +from cycode.cli.consts import PROGRAM_NAME +from cycode.cli.logger import logger +from cycode.cli.user_settings.configuration_manager import ConfigurationManager +from cycode.cli.utils.get_api_client import get_scan_cycode_client + +if TYPE_CHECKING: + from typer import Context + + +def get_cli_status(ctx: 'Context') -> CliStatus: + configuration_manager = ConfigurationManager() + + auth_info = get_authorization_info(ctx) + is_authenticated = auth_info is not None + + supported_modules_status = CliSupportedModulesStatus() + if is_authenticated: + try: + client = get_scan_cycode_client(ctx) + supported_modules_preferences = client.get_supported_modules_preferences() + + supported_modules_status.secret_scanning = supported_modules_preferences.secret_scanning + supported_modules_status.sca_scanning = supported_modules_preferences.sca_scanning + supported_modules_status.iac_scanning = supported_modules_preferences.iac_scanning + supported_modules_status.sast_scanning = supported_modules_preferences.sast_scanning + supported_modules_status.ai_large_language_model = supported_modules_preferences.ai_large_language_model + except Exception as e: + logger.debug('Failed to get supported modules preferences', exc_info=e) + + return CliStatus( + program=PROGRAM_NAME, + version=__version__, + os=platform.system(), + arch=platform.machine(), + python_version=platform.python_version(), + installation_id=configuration_manager.get_or_create_installation_id(), + app_url=configuration_manager.get_cycode_app_url(), + api_url=configuration_manager.get_cycode_api_url(), + is_authenticated=is_authenticated, + user_id=auth_info.user_id if auth_info else None, + tenant_id=auth_info.tenant_id if auth_info else None, + supported_modules=supported_modules_status, + ) diff --git a/cycode/cli/apps/status/models.py b/cycode/cli/apps/status/models.py new file mode 100644 index 00000000..82b9751a --- /dev/null +++ b/cycode/cli/apps/status/models.py @@ -0,0 +1,61 @@ +import json +from dataclasses import asdict, dataclass + + +class CliStatusBase: + def as_dict(self) -> dict[str, any]: + return asdict(self) + + def _get_text_message_part(self, key: str, value: any, intent: int = 0) -> str: + message_parts = [] + + intent_prefix = ' ' * intent * 2 + human_readable_key = key.replace('_', ' ').capitalize() + + if isinstance(value, dict): + message_parts.append(f'{intent_prefix}{human_readable_key}:') + for sub_key, sub_value in value.items(): + message_parts.append(self._get_text_message_part(sub_key, sub_value, intent=intent + 1)) + elif isinstance(value, (list, set, tuple)): + message_parts.append(f'{intent_prefix}{human_readable_key}:') + for index, sub_value in enumerate(value): + message_parts.append(self._get_text_message_part(f'#{index}', sub_value, intent=intent + 1)) + else: + message_parts.append(f'{intent_prefix}{human_readable_key}: {value}') + + return '\n'.join(message_parts) + + def as_text(self) -> str: + message_parts = [] + for key, value in self.as_dict().items(): + message_parts.append(self._get_text_message_part(key, value)) + + return '\n'.join(message_parts) + + def as_json(self) -> str: + return json.dumps(self.as_dict()) + + +@dataclass +class CliSupportedModulesStatus(CliStatusBase): + secret_scanning: bool = False + sca_scanning: bool = False + iac_scanning: bool = False + sast_scanning: bool = False + ai_large_language_model: bool = False + + +@dataclass +class CliStatus(CliStatusBase): + program: str + version: str + os: str + arch: str + python_version: str + installation_id: str + app_url: str + api_url: str + is_authenticated: bool + user_id: str = None + tenant_id: str = None + supported_modules: CliSupportedModulesStatus = None diff --git a/cycode/cli/apps/status/status_command.py b/cycode/cli/apps/status/status_command.py new file mode 100644 index 00000000..4654ef20 --- /dev/null +++ b/cycode/cli/apps/status/status_command.py @@ -0,0 +1,31 @@ +import typer + +from cycode.cli.apps.status.get_cli_status import get_cli_status +from cycode.cli.cli_types import OutputTypeOption +from cycode.cli.console import console + + +def status_command(ctx: typer.Context) -> None: + """:information_source: [bold cyan]Show Cycode CLI status and configuration.[/] + + This command displays the current status and configuration of the Cycode CLI, including: + * Authentication status: Whether you're logged in + * Version information: Current CLI version + * Configuration: Current API endpoints and settings + * System information: Operating system and environment details + + Output formats: + * Text: Human-readable format (default) + * JSON: Machine-readable format + + Example usage: + * `cycode status`: Show status in text format + * `cycode -o json status`: Show status in JSON format + """ + output = ctx.obj['output'] + + cli_status = get_cli_status(ctx) + if output == OutputTypeOption.JSON: + console.print_json(cli_status.as_json()) + else: + console.print(cli_status.as_text()) diff --git a/cycode/cli/apps/status/version_command.py b/cycode/cli/apps/status/version_command.py new file mode 100644 index 00000000..ef117fc7 --- /dev/null +++ b/cycode/cli/apps/status/version_command.py @@ -0,0 +1,10 @@ +import typer + +from cycode.cli.apps.status.status_command import status_command +from cycode.cli.console import console + + +def version_command(ctx: typer.Context) -> None: + console.print('[b yellow]This command is deprecated. Please use the "status" command instead.[/]') + console.line() + status_command(ctx) diff --git a/cycode/cli/auth/auth_command.py b/cycode/cli/auth/auth_command.py deleted file mode 100644 index f7605965..00000000 --- a/cycode/cli/auth/auth_command.py +++ /dev/null @@ -1,78 +0,0 @@ -import click -import traceback - -from cycode.cli.models import CliResult, CliErrors, CliError -from cycode.cli.printers import ConsolePrinter -from cycode.cli.auth.auth_manager import AuthManager -from cycode.cli.user_settings.credentials_manager import CredentialsManager -from cycode.cli.exceptions.custom_exceptions import AuthProcessError, NetworkError, HttpUnauthorizedError -from cycode.cyclient import logger -from cycode.cyclient.cycode_token_based_client import CycodeTokenBasedClient - - -@click.group(invoke_without_command=True) -@click.pass_context -def authenticate(context: click.Context): - """ Authenticates your machine to associate CLI with your cycode account """ - if context.invoked_subcommand is not None: - # if it is a subcommand do nothing - return - - try: - logger.debug('Starting authentication process') - - auth_manager = AuthManager() - auth_manager.authenticate() - - result = CliResult(success=True, message='Successfully logged into cycode') - ConsolePrinter(context).print_result(result) - except Exception as e: - _handle_exception(context, e) - - -@authenticate.command(name='check') -@click.pass_context -def authorization_check(context: click.Context): - """ Check your machine associating CLI with your cycode account """ - printer = ConsolePrinter(context) - - passed_auth_check_res = CliResult(success=True, message='You are authorized') - failed_auth_check_res = CliResult(success=False, message='You are not authorized') - - client_id, client_secret = CredentialsManager().get_credentials() - if not client_id or not client_secret: - return printer.print_result(failed_auth_check_res) - - try: - if CycodeTokenBasedClient(client_id, client_secret).api_token: - return printer.print_result(passed_auth_check_res) - except (NetworkError, HttpUnauthorizedError): - if context.obj['verbose']: - click.secho(f'Error: {traceback.format_exc()}', fg='red', nl=False) - - return printer.print_result(failed_auth_check_res) - - -def _handle_exception(context: click.Context, e: Exception): - if context.obj['verbose']: - click.secho(f'Error: {traceback.format_exc()}', fg='red', nl=False) - - errors: CliErrors = { - AuthProcessError: CliError( - code='auth_error', - message='Authentication failed. Please try again later using the command `cycode auth`' - ), - NetworkError: CliError( - code='cycode_error', - message='Authentication failed. Please try again later using the command `cycode auth`' - ), - } - - error = errors.get(type(e)) - if error: - return ConsolePrinter(context).print_error(error) - - if isinstance(e, click.ClickException): - raise e - - raise click.ClickException(str(e)) diff --git a/cycode/cli/ci_integrations.py b/cycode/cli/ci_integrations.py deleted file mode 100644 index a98abf4d..00000000 --- a/cycode/cli/ci_integrations.py +++ /dev/null @@ -1,61 +0,0 @@ -import os -import click - - -def github_action_range(): - before_sha = os.getenv("BEFORE_SHA") - push_base_sha = os.getenv("BASE_SHA") - pr_base_sha = os.getenv("PR_BASE_SHA") - default_branch = os.getenv("DEFAULT_BRANCH") - head_sha = os.getenv("GITHUB_SHA") - ref = os.getenv("GITHUB_REF") - - click.echo(f"{before_sha}, {push_base_sha}, {pr_base_sha}, {default_branch}, {head_sha}, {ref}") - if before_sha and before_sha != NO_COMMITS: - return f"{before_sha}..." - - return "..." - - # if pr_base_sha and pr_base_sha != FIRST_COMMIT: - # return f"{pr_base_sha}..." - # - # if push_base_sha and push_base_sha != "null": - # return f"{push_base_sha}..." - - -def circleci_range(): - before_sha = os.getenv("BEFORE_SHA") - current_sha = os.getenv("CURRENT_SHA") - commit_range = f"{before_sha}...{current_sha}" - click.echo(f"commit range: {commit_range}") - - if not commit_range.startswith('...'): - return commit_range - - commit_sha = os.getenv("CIRCLE_SHA1", "HEAD") - - return f'{commit_sha}~1...' - - -def gitlab_range(): - before_sha = os.getenv("CI_COMMIT_BEFORE_SHA") - commit_sha = os.getenv("CI_COMMIT_SHA", "HEAD") - - if before_sha and before_sha != NO_COMMITS: - return f"{before_sha}..." - - return f"{commit_sha}" - - -def get_commit_range(): - if os.getenv("GITHUB_ACTIONS"): - return github_action_range() - elif os.getenv("CIRCLECI"): - return circleci_range() - elif os.getenv("GITLAB_CI"): - return gitlab_range() - else: - raise click.ClickException("CI framework is not supported") - - -NO_COMMITS = "0000000000000000000000000000000000000000" diff --git a/cycode/cli/cli_types.py b/cycode/cli/cli_types.py new file mode 100644 index 00000000..ed277cc6 --- /dev/null +++ b/cycode/cli/cli_types.py @@ -0,0 +1,133 @@ +from enum import Enum + +from cycode.cli import consts + + +class StrEnum(str, Enum): + def __str__(self) -> str: + return self.value + + +class McpTransportOption(StrEnum): + STDIO = 'stdio' + SSE = 'sse' + STREAMABLE_HTTP = 'streamable-http' + + +class OutputTypeOption(StrEnum): + RICH = 'rich' + TEXT = 'text' + JSON = 'json' + TABLE = 'table' + + +class ExportTypeOption(StrEnum): + JSON = 'json' + HTML = 'html' + SVG = 'svg' + + +class ScanTypeOption(StrEnum): + SECRET = consts.SECRET_SCAN_TYPE + SCA = consts.SCA_SCAN_TYPE + IAC = consts.IAC_SCAN_TYPE + SAST = consts.SAST_SCAN_TYPE + + def __str__(self) -> str: + return self.value + + +class ScaScanTypeOption(StrEnum): + PACKAGE_VULNERABILITIES = 'package-vulnerabilities' + LICENSE_COMPLIANCE = 'license-compliance' + + +class SbomFormatOption(StrEnum): + SPDX_2_2 = 'spdx-2.2' + SPDX_2_3 = 'spdx-2.3' + CYCLONEDX_1_4 = 'cyclonedx-1.4' + CYCLONEDX_1_6 = 'cyclonedx-1.6' + + +class SbomOutputFormatOption(StrEnum): + JSON = 'json' + + +class BusinessImpactOption(StrEnum): + HIGH = 'High' + MEDIUM = 'Medium' + LOW = 'Low' + + +class SeverityOption(StrEnum): + INFO = 'info' + LOW = 'low' + MEDIUM = 'medium' + HIGH = 'high' + CRITICAL = 'critical' + + @classmethod + def _missing_(cls, value: str) -> str: + value = value.lower() + for member in cls: + if member.lower() == value: + return member + + return cls.INFO # fallback to INFO if no match is found + + @staticmethod + def get_member_weight(name: str) -> int: + return _SEVERITY_WEIGHTS.get(name.lower(), _SEVERITY_DEFAULT_WEIGHT) + + @staticmethod + def get_member_color(name: str) -> str: + return _SEVERITY_COLORS.get(name.lower(), _SEVERITY_DEFAULT_COLOR) + + @staticmethod + def get_member_emoji(name: str) -> str: + return _SEVERITY_EMOJIS.get(name.lower(), _SEVERITY_DEFAULT_EMOJI) + + @staticmethod + def get_member_unicode_emoji(name: str) -> str: + return _SEVERITY_UNICODE_EMOJIS.get(name.lower(), _SEVERITY_DEFAULT_UNICODE_EMOJI) + + def __rich__(self) -> str: + color = self.get_member_color(self.value) + return f'[{color}]{self.value.upper()}[/]' + + +_SEVERITY_DEFAULT_WEIGHT = -1 +_SEVERITY_WEIGHTS = { + SeverityOption.INFO.value: 0, + SeverityOption.LOW.value: 1, + SeverityOption.MEDIUM.value: 2, + SeverityOption.HIGH.value: 3, + SeverityOption.CRITICAL.value: 4, +} + +_SEVERITY_DEFAULT_COLOR = 'white' +_SEVERITY_COLORS = { + SeverityOption.INFO.value: 'deep_sky_blue1', + SeverityOption.LOW.value: 'gold1', + SeverityOption.MEDIUM.value: 'dark_orange', + SeverityOption.HIGH.value: 'red1', + SeverityOption.CRITICAL.value: 'red3', +} + +_SEVERITY_DEFAULT_EMOJI = ':white_circle:' +_SEVERITY_EMOJIS = { + SeverityOption.INFO.value: ':blue_circle:', + SeverityOption.LOW.value: ':yellow_circle:', + SeverityOption.MEDIUM.value: ':orange_circle:', + SeverityOption.HIGH.value: ':red_circle:', + SeverityOption.CRITICAL.value: ':exclamation_mark:', # double_exclamation_mark is not red +} + +_SEVERITY_DEFAULT_UNICODE_EMOJI = '⚪' +_SEVERITY_UNICODE_EMOJIS = { + SeverityOption.INFO.value: '🔵', + SeverityOption.LOW.value: '🟡', + SeverityOption.MEDIUM.value: '🟠', + SeverityOption.HIGH.value: '🔴', + SeverityOption.CRITICAL.value: '❗', +} diff --git a/cycode/cli/code_scanner.py b/cycode/cli/code_scanner.py deleted file mode 100644 index 0c0c996d..00000000 --- a/cycode/cli/code_scanner.py +++ /dev/null @@ -1,1046 +0,0 @@ -import click -import json -import logging -import os -import sys -import time -import traceback -from platform import platform -from uuid import uuid4, UUID -from git import Repo, NULL_TREE, InvalidGitRepositoryError -from sys import getsizeof - -from halo import Halo - -from cycode.cli.printers import ConsolePrinter -from cycode.cli.models import Document, DocumentDetections, Severity, CliError, CliErrors -from cycode.cli.ci_integrations import get_commit_range -from cycode.cli.consts import * -from cycode.cli.config import configuration_manager -from cycode.cli.utils.path_utils import is_sub_path, is_binary_file, get_file_size, get_relevant_files_in_path, \ - get_path_by_os, get_file_content -from cycode.cli.utils.string_utils import get_content_size, is_binary_content -from cycode.cli.utils.task_timer import TimeoutAfter -from cycode.cli.utils import scan_utils -from cycode.cli.user_settings.config_file_manager import ConfigFileManager -from cycode.cli.zip_file import InMemoryZip -from cycode.cli.exceptions.custom_exceptions import * -from cycode.cli.helpers import sca_code_scanner -from cycode.cyclient import logger -from cycode.cyclient.models import * - -start_scan_time = time.time() - - -@click.command() -@click.argument("path", nargs=1, type=click.STRING, required=True) -@click.option('--branch', '-b', - default=None, - help='Branch to scan, if not set scanning the default branch', - type=str, - required=False) -@click.pass_context -def scan_repository(context: click.Context, path, branch): - """ Scan git repository including its history """ - try: - logger.debug('Starting repository scan process, %s', {'path': path, 'branch': branch}) - context.obj["path"] = path - monitor = context.obj.get("monitor") - scan_type = context.obj["scan_type"] - if monitor and scan_type != SCA_SCAN_TYPE: - raise click.ClickException(f"Monitor flag is currently supported for SCA scan type only") - - documents_to_scan = [ - Document(obj.path if monitor else get_path_by_os(os.path.join(path, obj.path)), - obj.data_stream.read().decode('utf-8', errors='replace')) - for obj - in get_git_repository_tree_file_entries(path, branch)] - documents_to_scan = exclude_irrelevant_documents_to_scan(context, documents_to_scan) - perform_pre_scan_documents_actions(context, scan_type, documents_to_scan, False) - logger.debug('Found all relevant files for scanning %s', {'path': path, 'branch': branch}) - return scan_documents(context, documents_to_scan, is_git_diff=False, - scan_parameters=get_scan_parameters(context)) - except Exception as e: - _handle_exception(context, e) - - -@click.command() -@click.argument("path", - nargs=1, - type=click.STRING, - required=True) -@click.option("--commit_range", "-r", - help='Scan a commit range in this git repository, by default cycode scans all ' - 'commit history (example: HEAD~1)', - type=click.STRING, - default="--all", - required=False) -@click.pass_context -def scan_repository_commit_history(context: click.Context, path: str, commit_range: str): - """ Scan all the commits history in this git repository """ - try: - logger.debug('Starting commit history scan process, %s', {'path': path, 'commit_range': commit_range}) - return scan_commit_range(context, path=path, commit_range=commit_range) - except Exception as e: - _handle_exception(context, e) - - -def scan_commit_range(context: click.Context, path: str, commit_range: str, max_commits_count: Optional[int] = None): - scan_type = context.obj["scan_type"] - if scan_type not in COMMIT_RANGE_SCAN_SUPPORTED_SCAN_TYPES: - raise click.ClickException(f"Commit range scanning for {str.upper(scan_type)} is not supported") - - if scan_type == SCA_SCAN_TYPE: - return scan_sca_commit_range(context, path, commit_range) - - documents_to_scan = [] - commit_ids_to_scan = [] - - repo = Repo(path) - total_commits_count = repo.git.rev_list('--count', commit_range) - scanned_commits_count = 0 - logger.info(f'Calculating diffs for {total_commits_count} commits in the commit range {commit_range}') - for commit in repo.iter_commits(rev=commit_range): - if _does_reach_to_max_commits_to_scan_limit(commit_ids_to_scan, max_commits_count): - logger.info(f'Reached to max commits to scan count. Going to scan only {max_commits_count} last commits') - break - - if _should_update_progress(scanned_commits_count): - logger.info(f'Calculated diffs for {scanned_commits_count} out of {total_commits_count} commits') - - commit_id = commit.hexsha - commit_ids_to_scan.append(commit_id) - parent = commit.parents[0] if commit.parents else NULL_TREE - diff = commit.diff(parent, create_patch=True, R=True) - commit_documents_to_scan = [] - for blob in diff: - doc = Document(get_path_by_os(os.path.join(path, get_diff_file_path(blob))), - blob.diff.decode('utf-8', errors='replace'), True, unique_id=commit_id) - commit_documents_to_scan.append(doc) - - logger.debug('Found all relevant files in commit %s', - {'path': path, 'commit_range': commit_range, 'commit_id': commit_id}) - commit_documents_to_scan = exclude_irrelevant_documents_to_scan(context, commit_documents_to_scan) - documents_to_scan.extend(commit_documents_to_scan) - scanned_commits_count += 1 - - logger.debug('List of commit ids to scan, %s', {'commit_ids': commit_ids_to_scan}) - logger.info('Starting to scan commit range (It may take a few minutes)') - return scan_documents(context, documents_to_scan, is_git_diff=True, is_commit_range=True) - - -@click.command() -@click.pass_context -def scan_ci(context: click.Context): - """ Execute scan in a CI environment which relies on the - CYCODE_TOKEN and CYCODE_REPO_LOCATION environment variables """ - return scan_commit_range(context, path=os.getcwd(), commit_range=get_commit_range()) - - -@click.command() -@click.argument("path", nargs=1, type=click.STRING, required=True) -@click.pass_context -def scan_path(context: click.Context, path): - """ Scan the files in the path supplied in the command """ - logger.debug('Starting path scan process, %s', {'path': path}) - context.obj["path"] = path - files_to_scan = get_relevant_files_in_path(path=path, exclude_patterns=["**/.git/**", "**/.cycode/**"]) - files_to_scan = exclude_irrelevant_files(context, files_to_scan) - logger.debug('Found all relevant files for scanning %s', {'path': path, 'file_to_scan_count': len(files_to_scan)}) - return scan_disk_files(context, files_to_scan) - - -@click.command() -@click.argument("ignored_args", nargs=-1, type=click.UNPROCESSED) -@click.pass_context -def pre_commit_scan(context: click.Context, ignored_args: List[str]): - """ Use this command to scan the content that was not committed yet """ - scan_type = context.obj['scan_type'] - if scan_type == SCA_SCAN_TYPE: - return scan_sca_pre_commit(context) - - diff_files = Repo(os.getcwd()).index.diff("HEAD", create_patch=True, R=True) - documents_to_scan = [Document(get_path_by_os(get_diff_file_path(file)), get_diff_file_content(file)) - for file in diff_files] - documents_to_scan = exclude_irrelevant_documents_to_scan(context, documents_to_scan) - return scan_documents(context, documents_to_scan, is_git_diff=True) - - -@click.command() -@click.argument("ignored_args", nargs=-1, type=click.UNPROCESSED) -@click.pass_context -def pre_receive_scan(context: click.Context, ignored_args: List[str]): - """ Use this command to scan commits on server side before pushing them to the repository """ - try: - scan_type = context.obj['scan_type'] - if scan_type != SECRET_SCAN_TYPE: - raise click.ClickException(f"Commit range scanning for {str.upper(scan_type)} is not supported") - - if should_skip_pre_receive_scan(): - logger.info( - "A scan has been skipped as per your request." - " Please note that this may leave your system vulnerable to secrets that have not been detected") - return - - if is_verbose_mode_requested_in_pre_receive_scan(): - enable_verbose_mode(context) - logger.debug('Verbose mode enabled, all log levels will be displayed') - - command_scan_type = context.info_name - timeout = configuration_manager.get_pre_receive_command_timeout(command_scan_type) - with TimeoutAfter(timeout): - if scan_type not in COMMIT_RANGE_SCAN_SUPPORTED_SCAN_TYPES: - raise click.ClickException(f"Commit range scanning for {str.upper(scan_type)} is not supported") - - branch_update_details = parse_pre_receive_input() - commit_range = calculate_pre_receive_commit_range(branch_update_details) - if not commit_range: - logger.info('No new commits found for pushed branch, %s', - {'branch_update_details': branch_update_details}) - return - - max_commits_to_scan = configuration_manager.get_pre_receive_max_commits_to_scan_count(command_scan_type) - scan_commit_range(context, os.getcwd(), commit_range, max_commits_count=max_commits_to_scan) - perform_post_pre_receive_scan_actions(context) - except Exception as e: - _handle_exception(context, e) - - -def scan_sca_pre_commit(context: click.Context): - scan_parameters = get_default_scan_parameters(context) - git_head_documents, pre_committed_documents = get_pre_commit_modified_documents() - git_head_documents = exclude_irrelevant_documents_to_scan(context, git_head_documents) - pre_committed_documents = exclude_irrelevant_documents_to_scan(context, pre_committed_documents) - sca_code_scanner.perform_pre_hook_range_scan_actions(git_head_documents, pre_committed_documents) - return scan_commit_range_documents(context, git_head_documents, pre_committed_documents, - scan_parameters, - configuration_manager.get_sca_pre_commit_timeout_in_seconds()) - - -def scan_sca_commit_range(context: click.Context, path: str, commit_range: str): - context.obj["path"] = path - scan_parameters = get_scan_parameters(context) - from_commit_rev, to_commit_rev = parse_commit_range(commit_range, path) - from_commit_documents, to_commit_documents = \ - get_commit_range_modified_documents(path, from_commit_rev, to_commit_rev) - from_commit_documents = exclude_irrelevant_documents_to_scan(context, from_commit_documents) - to_commit_documents = exclude_irrelevant_documents_to_scan(context, to_commit_documents) - sca_code_scanner.perform_pre_commit_range_scan_actions(path, from_commit_documents, from_commit_rev, - to_commit_documents, to_commit_rev) - - return scan_commit_range_documents(context, from_commit_documents, to_commit_documents, - scan_parameters=scan_parameters) - - -def scan_disk_files(context: click.Context, paths: List[str]): - scan_parameters = get_scan_parameters(context) - scan_type = context.obj['scan_type'] - is_git_diff = False - documents: List[Document] = [] - for path in paths: - with open(path, "r", encoding="utf-8") as f: - content = f.read() - documents.append(Document(path, content, is_git_diff)) - - perform_pre_scan_documents_actions(context, scan_type, documents, is_git_diff) - return scan_documents(context, documents, is_git_diff=is_git_diff, scan_parameters=scan_parameters) - - -def scan_documents(context: click.Context, documents_to_scan: List[Document], is_git_diff: bool = False, - is_commit_range: bool = False, scan_parameters: dict = None): - cycode_client = context.obj["client"] - scan_type = context.obj["scan_type"] - severity_threshold = context.obj["severity_threshold"] - command_scan_type = context.info_name - error_message = None - all_detections_count = 0 - output_detections_count = 0 - scan_id = _get_scan_id(context) - zipped_documents = InMemoryZip() - - try: - logger.debug("Preparing local files") - zipped_documents = zip_documents_to_scan(scan_type, zipped_documents, documents_to_scan) - - scan_result = perform_scan(context, cycode_client, zipped_documents, scan_type, scan_id, is_git_diff, - is_commit_range, - scan_parameters) - - all_detections_count, output_detections_count = \ - handle_scan_result(context, scan_result, command_scan_type, scan_type, severity_threshold, - documents_to_scan) - scan_completed = True - except Exception as e: - _handle_exception(context, e) - error_message = str(e) - scan_completed = False - - zip_file_size = getsizeof(zipped_documents.in_memory_zip) - logger.debug('Finished scan process, %s', - {'all_violations_count': all_detections_count, 'relevant_violations_count': output_detections_count, - 'scan_id': str(scan_id), 'zip_file_size': zip_file_size}) - _report_scan_status(context, scan_type, str(scan_id), scan_completed, output_detections_count, - all_detections_count, len(documents_to_scan), zip_file_size, command_scan_type, error_message) - - -def scan_commit_range_documents(context: click.Context, from_documents_to_scan: List[Document], - to_documents_to_scan: List[Document], scan_parameters: dict = None, - timeout: int = None): - cycode_client = context.obj["client"] - scan_type = context.obj["scan_type"] - severity_threshold = context.obj["severity_threshold"] - scan_command_type = context.info_name - error_message = None - all_detections_count = 0 - output_detections_count = 0 - scan_id = _get_scan_id(context) - from_commit_zipped_documents = InMemoryZip() - to_commit_zipped_documents = InMemoryZip() - - try: - scan_result = init_default_scan_result(str(scan_id)) - if should_scan_documents(from_documents_to_scan, to_documents_to_scan): - logger.debug("Preparing from-commit zip") - from_commit_zipped_documents = zip_documents_to_scan(scan_type, from_commit_zipped_documents, - from_documents_to_scan) - logger.debug("Preparing to-commit zip") - to_commit_zipped_documents = zip_documents_to_scan(scan_type, to_commit_zipped_documents, - to_documents_to_scan) - scan_result = perform_commit_range_scan_async(context, cycode_client, from_commit_zipped_documents, - to_commit_zipped_documents, scan_type, scan_parameters, - timeout) - all_detections_count, output_detections_count = \ - handle_scan_result(context, scan_result, scan_command_type, scan_type, severity_threshold, - to_documents_to_scan) - scan_completed = True - except Exception as e: - _handle_exception(context, e) - error_message = str(e) - scan_completed = False - - zip_file_size = getsizeof(from_commit_zipped_documents.in_memory_zip) + getsizeof( - to_commit_zipped_documents.in_memory_zip) - logger.debug('Finished scan process, %s', - {'all_violations_count': all_detections_count, 'relevant_violations_count': output_detections_count, - 'scan_id': str(scan_id), 'zip_file_size': zip_file_size}) - _report_scan_status(context, scan_type, str(scan_id), scan_completed, output_detections_count, - all_detections_count, len(to_documents_to_scan), zip_file_size, scan_command_type, - error_message) - - -def should_scan_documents(from_documents_to_scan: List[Document], to_documents_to_scan: List[Document]) -> bool: - return len(from_documents_to_scan) > 0 or len(to_documents_to_scan) > 0 - - -def handle_scan_result(context, scan_result, command_scan_type, scan_type, severity_threshold, to_documents_to_scan): - document_detections_list = enrich_scan_result(scan_result, to_documents_to_scan) - relevant_document_detections_list = exclude_irrelevant_scan_results(document_detections_list, scan_type, - command_scan_type, severity_threshold) - context.obj['report_url'] = scan_result.report_url - print_results(context, relevant_document_detections_list) - context.obj['issue_detected'] = len(relevant_document_detections_list) > 0 - all_detections_count = sum( - [len(document_detections.detections) for document_detections in document_detections_list]) - output_detections_count = sum( - [len(document_detections.detections) for document_detections in relevant_document_detections_list]) - return all_detections_count, output_detections_count - - -def perform_pre_scan_documents_actions(context: click.Context, scan_type: str, documents_to_scan: List[Document], - is_git_diff: bool = False): - if scan_type == SCA_SCAN_TYPE: - logger.debug( - f"Perform pre scan document actions") - sca_code_scanner.add_dependencies_tree_document(context, documents_to_scan, is_git_diff) - - -def zip_documents_to_scan(scan_type: str, zip: InMemoryZip, documents: List[Document]): - start_zip_creation_time = time.time() - - for index, document in enumerate(documents): - zip_file_size = getsizeof(zip.in_memory_zip) - validate_zip_file_size(scan_type, zip_file_size) - - logger.debug('adding file to zip, %s', - {'index': index, 'filename': document.path, 'unique_id': document.unique_id}) - zip.append(document.path, document.unique_id, document.content) - zip.close() - - end_zip_creation_time = time.time() - zip_creation_time = int(end_zip_creation_time - start_zip_creation_time) - logger.debug('finished to create zip file, %s', {'zip_creation_time': zip_creation_time}) - return zip - - -def validate_zip_file_size(scan_type, zip_file_size): - if scan_type == SCA_SCAN_TYPE: - if zip_file_size > SCA_ZIP_MAX_SIZE_LIMIT_IN_BYTES: - raise ZipTooLargeError(SCA_ZIP_MAX_SIZE_LIMIT_IN_BYTES) - else: - if zip_file_size > ZIP_MAX_SIZE_LIMIT_IN_BYTES: - raise ZipTooLargeError(ZIP_MAX_SIZE_LIMIT_IN_BYTES) - - -def perform_scan(context, cycode_client, zipped_documents: InMemoryZip, scan_type: str, scan_id: UUID, - is_git_diff: bool, - is_commit_range: bool, scan_parameters: dict): - if scan_type == SCA_SCAN_TYPE or scan_type == SAST_SCAN_TYPE: - return perform_scan_async(context, cycode_client, zipped_documents, scan_type, scan_parameters) - - scan_result = cycode_client.commit_range_zipped_file_scan(scan_type, zipped_documents, scan_id) \ - if is_commit_range else cycode_client.zipped_file_scan(scan_type, zipped_documents, scan_id, - scan_parameters, is_git_diff) - - return scan_result - - -def perform_scan_async(context: click.Context, cycode_client, zipped_documents: InMemoryZip, scan_type: str, - scan_parameters: dict) -> ZippedFileScanResult: - scan_async_result = cycode_client.zipped_file_scan_async(zipped_documents, scan_type, scan_parameters) - logger.debug("scan request has been triggered successfully, scan id: %s", scan_async_result.scan_id) - - return poll_scan_results(context, cycode_client, scan_async_result.scan_id) - - -def perform_commit_range_scan_async(context: click.Context, cycode_client, from_commit_zipped_documents: InMemoryZip, - to_commit_zipped_documents: InMemoryZip, scan_type: str, - scan_parameters: dict, timeout: int = None) -> ZippedFileScanResult: - scan_async_result = \ - cycode_client.multiple_zipped_file_scan_async(from_commit_zipped_documents, to_commit_zipped_documents, - scan_type, scan_parameters) - logger.debug("scan request has been triggered successfully, scan id: %s", scan_async_result.scan_id) - return poll_scan_results(context, cycode_client, scan_async_result.scan_id, timeout) - - -def poll_scan_results(context: click.Context, cycode_client, scan_id: str, polling_timeout: int = None): - if polling_timeout is None: - polling_timeout = configuration_manager.get_scan_polling_timeout_in_seconds() - - last_scan_update_at = None - end_polling_time = time.time() + polling_timeout - spinner = Halo(spinner='dots') - spinner.start("Scan in progress") - while time.time() < end_polling_time: - scan_details = cycode_client.get_scan_details(scan_id) - if scan_details.scan_update_at is not None and scan_details.scan_update_at != last_scan_update_at: - click.echo() - last_scan_update_at = scan_details.scan_update_at - print_scan_details(scan_details) - if scan_details.scan_status == SCAN_STATUS_COMPLETED: - spinner.succeed() - return _get_scan_result(cycode_client, scan_id, scan_details) - if scan_details.scan_status == SCAN_STATUS_ERROR: - spinner.fail() - raise ScanAsyncError(f'error occurred while trying to scan zip file. {scan_details.message}') - time.sleep(SCAN_POLLING_WAIT_INTERVAL_IN_SECONDS) - - spinner.stop_and_persist(symbol="⏰".encode('utf-8'), text='Timeout') - raise ScanAsyncError(f'Failed to complete scan after {polling_timeout} seconds') - - -def print_scan_details(scan_details_response: ScanDetailsResponse): - logger.info(f"Scan update: (scan_id: {scan_details_response.id})") - logger.info(f"Scan status: {scan_details_response.scan_status}") - if scan_details_response.message is not None: - logger.info(f"Scan message: {scan_details_response.message}") - - -def print_results(context: click.Context, document_detections_list: List[DocumentDetections]) -> None: - printer = ConsolePrinter(context) - printer.print_scan_results(document_detections_list) - - -def enrich_scan_result( - scan_result: ZippedFileScanResult, documents_to_scan: List[Document] -) -> List[DocumentDetections]: - logger.debug('enriching scan result') - document_detections_list = [] - for detections_per_file in scan_result.detections_per_file: - file_name = get_path_by_os(detections_per_file.file_name) - commit_id = detections_per_file.commit_id - logger.debug("going to find document of violated file, %s", {'file_name': file_name, 'commit_id': commit_id}) - document = _get_document_by_file_name(documents_to_scan, file_name, commit_id) - document_detections_list.append( - DocumentDetections(document=document, detections=detections_per_file.detections)) - - return document_detections_list - - -def exclude_irrelevant_scan_results(document_detections_list: List[DocumentDetections], scan_type: str, - command_scan_type: str, severity_threshold: str) -> List[DocumentDetections]: - relevant_document_detections_list = [] - for document_detections in document_detections_list: - relevant_detections = exclude_irrelevant_detections(scan_type, command_scan_type, severity_threshold, - document_detections.detections) - if relevant_detections: - relevant_document_detections_list.append(DocumentDetections(document=document_detections.document, - detections=relevant_detections)) - - return relevant_document_detections_list - - -def parse_pre_receive_input() -> str: - """ - Parsing input to pushed branch update details - - Example input: - old_value new_value refname - ----------------------------------------------- - 0000000000000000000000000000000000000000 9cf90954ef26e7c58284f8ebf7dcd0fcf711152a refs/heads/main - 973a96d3e925b65941f7c47fa16129f1577d499f 0000000000000000000000000000000000000000 refs/heads/feature-branch - 59564ef68745bca38c42fc57a7822efd519a6bd9 3378e52dcfa47fb11ce3a4a520bea5f85d5d0bf3 refs/heads/develop - - :return: first branch update details (input's first line) - """ - pre_receive_input = sys.stdin.read().strip() - if not pre_receive_input: - raise ValueError( - "Pre receive input was not found. Make sure that you are using this command only in pre-receive hook") - - # each line represents a branch update request, handle the first one only - # TODO support case of multiple update branch requests - branch_update_details = pre_receive_input.splitlines()[0] - return branch_update_details - - -def calculate_pre_receive_commit_range(branch_update_details: str) -> Optional[str]: - end_commit = get_end_commit_from_branch_update_details(branch_update_details) - - # branch is deleted, no need to perform scan - if end_commit == EMPTY_COMMIT_SHA: - return - - start_commit = get_oldest_unupdated_commit_for_branch(end_commit) - - # no new commit to update found - if not start_commit: - return - - return f'{start_commit}~1...{end_commit}' - - -def get_end_commit_from_branch_update_details(update_details: str) -> str: - # update details pattern: - _, end_commit, _ = update_details.split() - return end_commit - - -def get_oldest_unupdated_commit_for_branch(commit: str) -> Optional[str]: - # get a list of commits by chronological order that are not in the remote repository yet - # more info about rev-list command: https://git-scm.com/docs/git-rev-list - not_updated_commits = Repo(os.getcwd()).git.rev_list(commit, '--topo-order', '--reverse', '--not', '--all') - commits = not_updated_commits.splitlines() - if not commits: - return None - return commits[0] - - -def get_diff_file_path(file): - return file.b_path if file.b_path else file.a_path - - -def get_diff_file_content(file): - return file.diff.decode('utf-8', errors='replace') - - -def should_process_git_object(obj, depth): - return obj.type == 'blob' and obj.size > 0 - - -def get_git_repository_tree_file_entries(path: str, branch: str): - return Repo(path).tree(branch).traverse(predicate=should_process_git_object) - - -def get_default_scan_parameters(context: click.Context) -> dict: - return { - "monitor": context.obj.get("monitor"), - "report": context.obj.get("report"), - "package_vulnerabilities": context.obj.get("package-vulnerabilities"), - "license_compliance": context.obj.get("license-compliance") - } - - -def get_scan_parameters(context: click.Context) -> dict: - path = context.obj["path"] - scan_parameters = get_default_scan_parameters(context) - remote_url = try_get_git_remote_url(path) - if remote_url: - context.obj["remote_url"] = remote_url - scan_parameters.update(remote_url) - return scan_parameters - - -def try_get_git_remote_url(path: str) -> Optional[dict]: - try: - git_remote_url = Repo(path).remotes[0].config_reader.get('url') - return { - 'remote_url': git_remote_url, - } - except Exception as e: - logger.debug('Failed to get git remote URL. %s', {'exception_message': str(e)}) - return None - - -def exclude_irrelevant_documents_to_scan(context: click.Context, documents_to_scan: List[Document]) -> \ - List[Document]: - scan_type = context.obj['scan_type'] - logger.debug("excluding irrelevant documents to scan") - return [document for document in documents_to_scan if - _is_relevant_document_to_scan(scan_type, document.path, document.content)] - - -def exclude_irrelevant_files(context: click.Context, filenames: List[str]) -> List[str]: - scan_type = context.obj['scan_type'] - return [filename for filename in filenames if _is_relevant_file_to_scan(scan_type, filename)] - - -def exclude_irrelevant_detections(scan_type: str, command_scan_type: str, severity_threshold: str, detections) -> List: - relevant_detections = exclude_detections_by_exclusions_configuration(scan_type, detections) - relevant_detections = exclude_detections_by_scan_type(scan_type, command_scan_type, relevant_detections) - relevant_detections = exclude_detections_by_severity(scan_type, severity_threshold, relevant_detections) - - return relevant_detections - - -def exclude_detections_by_severity(scan_type: str, severity_threshold: str, detections) -> List: - if scan_type != SCA_SCAN_TYPE or severity_threshold is None: - return detections - - return [detection for detection in detections if - _does_severity_match_severity_threshold(detection.detection_details.get('advisory_severity'), - severity_threshold)] - - -def exclude_detections_by_scan_type(scan_type: str, command_scan_type: str, detections) -> List: - if command_scan_type == PRE_COMMIT_COMMAND_SCAN_TYPE: - return exclude_detections_in_deleted_lines(detections) - - if command_scan_type in COMMIT_RANGE_BASED_COMMAND_SCAN_TYPES \ - and scan_type == SECRET_SCAN_TYPE \ - and configuration_manager.get_should_exclude_detections_in_deleted_lines(command_scan_type): - return exclude_detections_in_deleted_lines(detections) - return detections - - -def exclude_detections_in_deleted_lines(detections) -> List: - return [detection for detection in detections if detection.detection_details.get('line_type') != 'Removed'] - - -def exclude_detections_by_exclusions_configuration(scan_type: str, detections) -> List: - exclusions = configuration_manager.get_exclusions_by_scan_type(scan_type) - return [detection for detection in detections if not _should_exclude_detection(detection, exclusions)] - - -def get_pre_commit_modified_documents(): - repo = Repo(os.getcwd()) - diff_files = repo.index.diff(GIT_HEAD_COMMIT_REV, create_patch=True, R=True) - git_head_documents = [] - pre_committed_documents = [] - for file in diff_files: - diff_file_path = get_diff_file_path(file) - file_path = get_path_by_os(diff_file_path) - - file_content = sca_code_scanner.get_file_content_from_commit(repo, GIT_HEAD_COMMIT_REV, diff_file_path) - if file_content is not None: - git_head_documents.append(Document(file_path, file_content)) - - if os.path.exists(file_path): - file_content = get_file_content(file_path) - pre_committed_documents.append(Document(file_path, file_content)) - - return git_head_documents, pre_committed_documents - - -def get_commit_range_modified_documents(path: str, from_commit_rev: str, to_commit_rev: str) -> ( - List[Document], List[Document]): - from_commit_documents = [] - to_commit_documents = [] - repo = Repo(path) - diff = repo.commit(from_commit_rev).diff(to_commit_rev) - modified_files_diff = [change for change in diff if change.change_type != COMMIT_DIFF_DELETED_FILE_CHANGE_TYPE] - for blob in modified_files_diff: - diff_file_path = get_diff_file_path(blob) - file_path = get_path_by_os(diff_file_path) - - file_content = sca_code_scanner.get_file_content_from_commit(repo, from_commit_rev, diff_file_path) - if file_content is not None: - from_commit_documents.append(Document(file_path, file_content)) - - file_content = sca_code_scanner.get_file_content_from_commit(repo, to_commit_rev, diff_file_path) - if file_content is not None: - to_commit_documents.append(Document(file_path, file_content)) - - return from_commit_documents, to_commit_documents - - -def _should_exclude_detection(detection, exclusions: Dict) -> bool: - exclusions_by_value = exclusions.get(EXCLUSIONS_BY_VALUE_SECTION_NAME, []) - if _is_detection_sha_configured_in_exclusions(detection, exclusions_by_value): - logger.debug('Going to ignore violations because is in the values to ignore list, %s', - {'sha': detection.detection_details.get('sha512', '')}) - return True - - exclusions_by_sha = exclusions.get(EXCLUSIONS_BY_SHA_SECTION_NAME, []) - if _is_detection_sha_configured_in_exclusions(detection, exclusions_by_sha): - logger.debug('Going to ignore violations because is in the shas to ignore list, %s', - {'sha': detection.detection_details.get('sha512', '')}) - return True - - exclusions_by_rule = exclusions.get(EXCLUSIONS_BY_RULE_SECTION_NAME, []) - if exclusions_by_rule: - detection_rule = detection.detection_rule_id - if detection_rule in exclusions_by_rule: - logger.debug('Going to ignore violations because is in the shas to ignore list, %s', - {'detection_rule': detection_rule}) - return True - - exclusions_by_package = exclusions.get(EXCLUSIONS_BY_PACKAGE_SECTION_NAME, []) - if exclusions_by_package: - package = _get_package_name(detection) - if package in exclusions_by_package: - logger.debug('Going to ignore violations because is in the packages to ignore list, %s', - {'package': package}) - return True - - return False - - -def _is_detection_sha_configured_in_exclusions(detection, exclusions: List[str]) -> bool: - detection_sha = detection.detection_details.get('sha512', '') - return detection_sha in exclusions - - -def _is_path_configured_in_exclusions(scan_type: str, file_path: str) -> bool: - exclusions_by_path = configuration_manager.get_exclusions_by_scan_type(scan_type).get( - EXCLUSIONS_BY_PATH_SECTION_NAME, []) - for exclusion_path in exclusions_by_path: - if is_sub_path(exclusion_path, file_path): - return True - return False - - -def _get_package_name(detection) -> str: - package_name = detection.detection_details.get('vulnerable_component', '') - package_version = detection.detection_details.get('vulnerable_component_version', '') - - if package_name == '': - package_name = detection.detection_details.get('package_name', '') - package_version = detection.detection_details.get('package_version', '') - - return f'{package_name}@{package_version}' - - -def _is_file_relevant_for_sca_scan(filename: str) -> bool: - if any([sca_excluded_path in filename for sca_excluded_path in SCA_EXCLUDED_PATHS]): - logger.debug("file is irrelevant because it is from node_modules's inner path, %s", - {'filename': filename}) - return False - return True - - -def _is_relevant_file_to_scan(scan_type: str, filename: str) -> bool: - if _is_subpath_of_cycode_configuration_folder(filename): - logger.debug("file is irrelevant because it is in cycode configuration directory, %s", - {'filename': filename}) - return False - - if _is_path_configured_in_exclusions(scan_type, filename): - logger.debug("file is irrelevant because the file path is in the ignore paths list, %s", - {'filename': filename}) - return False - - if not _is_file_extension_supported(scan_type, filename): - logger.debug("file is irrelevant because the file extension is not supported, %s", - {'filename': filename}) - return False - - if is_binary_file(filename): - logger.debug("file is irrelevant because it is binary file, %s", - {'filename': filename}) - return False - - if scan_type != SCA_SCAN_TYPE and _does_file_exceed_max_size_limit(filename): - logger.debug("file is irrelevant because its exceeded max size limit, %s", - {'filename': filename}) - return False - - if scan_type == SCA_SCAN_TYPE and not _is_file_relevant_for_sca_scan(filename): - return False - - return True - - -def _is_relevant_document_to_scan(scan_type: str, filename: str, content: str) -> bool: - if _is_subpath_of_cycode_configuration_folder(filename): - logger.debug("document is irrelevant because it is in cycode configuration directory, %s", - {'filename': filename}) - return False - - if _is_path_configured_in_exclusions(scan_type, filename): - logger.debug("document is irrelevant because the document path is in the ignore paths list, %s", - {'filename': filename}) - return False - - if not _is_file_extension_supported(scan_type, filename): - logger.debug("document is irrelevant because the file extension is not supported, %s", - {'filename': filename}) - return False - - if is_binary_content(content): - logger.debug("document is irrelevant because it is binary, %s", - {'filename': filename}) - return False - - if scan_type != SCA_SCAN_TYPE and _does_document_exceed_max_size_limit(content): - logger.debug("document is irrelevant because its exceeded max size limit, %s", - {'filename': filename}) - return False - return True - - -def _is_file_extension_supported(scan_type: str, filename: str) -> bool: - if scan_type == INFRA_CONFIGURATION_SCAN_TYPE: - return any(filename.lower().endswith(supported_file_extension) for supported_file_extension in - INFRA_CONFIGURATION_SCAN_SUPPORTED_FILES) - if scan_type == SCA_SCAN_TYPE: - return any(filename.lower().endswith(supported_file) for supported_file in - SCA_CONFIGURATION_SCAN_SUPPORTED_FILES) - return all(not filename.lower().endswith(file_extension_to_ignore) for file_extension_to_ignore in - SECRET_SCAN_FILE_EXTENSIONS_TO_IGNORE) - - -def _does_file_exceed_max_size_limit(filename: str) -> bool: - return FILE_MAX_SIZE_LIMIT_IN_BYTES < get_file_size(filename) - - -def _get_document_by_file_name(documents: List[Document], file_name: str, unique_id: Optional[str] = None) \ - -> Optional[Document]: - return next( - (document for document in documents if _normalize_file_path(document.path) == _normalize_file_path(file_name) - and document.unique_id == unique_id), None) - - -def _does_document_exceed_max_size_limit(content: str) -> bool: - return FILE_MAX_SIZE_LIMIT_IN_BYTES < get_content_size(content) - - -def _is_subpath_of_cycode_configuration_folder(filename: str) -> bool: - return is_sub_path(configuration_manager.global_config_file_manager.get_config_directory_path(), filename) \ - or is_sub_path(configuration_manager.local_config_file_manager.get_config_directory_path(), filename) \ - or filename.endswith(ConfigFileManager.get_config_file_route()) - - -def _handle_exception(context: click.Context, e: Exception): - context.obj['did_fail'] = True - - if context.obj['verbose']: - click.secho(f'Error: {traceback.format_exc()}', fg='red', nl=False) - - errors: CliErrors = { - NetworkError: CliError( - soft_fail=True, - code='cycode_error', - message='Cycode was unable to complete this scan. ' - 'Please try again by executing the `cycode scan` command' - ), - ScanAsyncError: CliError( - soft_fail=True, - code='scan_error', - message='Cycode was unable to complete this scan. ' - 'Please try again by executing the `cycode scan` command' - ), - HttpUnauthorizedError: CliError( - soft_fail=True, - code='auth_error', - message='Unable to authenticate to Cycode, your token is either invalid or has expired. ' - 'Please re-generate your token and reconfigure it by running the `cycode configure` command' - ), - ZipTooLargeError: CliError( - soft_fail=True, - code='zip_too_large_error', - message='The path you attempted to scan exceeds the current maximum scanning size cap (10MB). ' - 'Please try ignoring irrelevant paths using the ‘cycode ignore --by-path’ command ' - 'and execute the scan again' - ), - InvalidGitRepositoryError: CliError( - soft_fail=False, - code='invalid_git_error', - message='The path you supplied does not correlate to a git repository. ' - 'Should you still wish to scan this path, use: ‘cycode scan path ’' - ), - } - - if type(e) in errors: - error = errors[type(e)] - - if error.soft_fail is True: - context.obj['soft_fail'] = True - - return ConsolePrinter(context).print_error(error) - - if isinstance(e, click.ClickException): - raise e - - raise click.ClickException(str(e)) - - -def _report_scan_status(context: click.Context, scan_type: str, scan_id: str, scan_completed: bool, - output_detections_count: int, all_detections_count: int, files_to_scan_count: int, - zip_size: int, command_scan_type: str, error_message: Optional[str]): - try: - cycode_client = context.obj["client"] - end_scan_time = time.time() - scan_status = { - 'zip_size': zip_size, - 'execution_time': int(end_scan_time - start_scan_time), - 'output_detections_count': output_detections_count, - 'all_detections_count': all_detections_count, - 'scannable_files_count': files_to_scan_count, - 'status': 'Completed' if scan_completed else 'Error', - 'scan_command_type': command_scan_type, - 'operation_system': platform(), - 'error_message': error_message, - 'scan_type': scan_type - } - - cycode_client.report_scan_status(scan_type, scan_id, scan_status) - except Exception as e: - logger.debug('Failed to report scan status, %s', {'exception_message': str(e)}) - pass - - -def _get_scan_id(context: click.Context): - scan_id = uuid4() - context.obj['scan_id'] = scan_id - return scan_id - - -def _does_severity_match_severity_threshold(severity: str, severity_threshold: str) -> bool: - detection_severity_value = Severity.try_get_value(severity) - if detection_severity_value is None: - return True - - return detection_severity_value >= Severity.try_get_value(severity_threshold) - - -def _get_scan_result(cycode_client, scan_id: str, scan_details: ScanDetailsResponse) -> ZippedFileScanResult: - scan_result = init_default_scan_result(scan_id, scan_details.metadata) - if not scan_details.detections_count: - return scan_result - - wait_for_detections_creation(cycode_client, scan_id, scan_details.detections_count) - scan_detections = cycode_client.get_scan_detections(scan_id) - scan_result.detections_per_file = _map_detections_per_file(scan_detections) - scan_result.did_detect = True - return scan_result - - -def init_default_scan_result(scan_id: str, scan_metadata: str = None) -> ZippedFileScanResult: - return ZippedFileScanResult(did_detect=False, detections_per_file=[], - scan_id=scan_id, - report_url=_try_get_report_url(scan_metadata)) - - -def _try_get_report_url(metadata: str) -> Optional[str]: - if metadata is None: - return None - try: - metadata = json.loads(metadata) - return metadata.get('report_url') - except ValueError: - return None - - -def wait_for_detections_creation(cycode_client, scan_id: str, expected_detections_count: int): - logger.debug("waiting for detections to be created") - scan_persisted_detections_count = 0 - end_polling_time = time.time() + DETECTIONS_COUNT_VERIFICATION_TIMEOUT_IN_SECONDS - spinner = Halo(spinner='dots') - spinner.start("Please wait until printing scan result...") - while time.time() < end_polling_time: - scan_persisted_detections_count = cycode_client.get_scan_detections_count(scan_id) - if scan_persisted_detections_count == expected_detections_count: - spinner.succeed() - return - time.sleep(DETECTIONS_COUNT_VERIFICATION_WAIT_INTERVAL_IN_SECONDS) - spinner.stop_and_persist(symbol="⏰".encode('utf-8'), text='Timeout') - logger.debug("%i detections has been created", scan_persisted_detections_count) - - -def _map_detections_per_file(detections) -> List[DetectionsPerFile]: - detections_per_files = {} - for detection in detections: - try: - detection['message'] = detection['correlation_message'] - file_name = _get_file_name_from_detection(detection) - if file_name is None: - logger.debug("file name is missing from detection with id %s", detection.get('id')) - continue - if detections_per_files.get(file_name) is None: - detections_per_files[file_name] = [DetectionSchema().load(detection)] - else: - detections_per_files[file_name].append(DetectionSchema().load(detection)) - except Exception as e: - logger.debug("Failed to parse detection: %s", str(e)) - continue - - return [DetectionsPerFile(file_name=file_name, detections=file_detections) - for file_name, file_detections in detections_per_files.items()] - - -def _get_file_name_from_detection(detection): - if detection['category'] == "SAST": - return detection['detection_details']['file_path'] - - return detection['detection_details']['file_name'] - - -def _does_reach_to_max_commits_to_scan_limit(commit_ids: List, max_commits_count: Optional[int]) -> bool: - return max_commits_count is not None and len(commit_ids) >= max_commits_count - - -def _should_update_progress(scanned_commits_count: int) -> bool: - return scanned_commits_count and scanned_commits_count % PROGRESS_UPDATE_COMMITS_INTERVAL == 0 - - -def parse_commit_range(commit_range: str, path: str) -> (str, str): - from_commit_rev = None - to_commit_rev = None - for commit in Repo(path).iter_commits(rev=commit_range): - if not to_commit_rev: - to_commit_rev = commit.hexsha - from_commit_rev = commit.hexsha - - return from_commit_rev, to_commit_rev - - -def _normalize_file_path(path: str): - if path.startswith("/"): - return path[1:] - if path.startswith("./"): - return path[2:] - return path - - -def perform_post_pre_receive_scan_actions(context: click.Context): - if scan_utils.is_scan_failed(context): - click.echo(PRE_RECEIVE_REMEDIATION_MESSAGE) - - -def enable_verbose_mode(context: click.Context): - context.obj["verbose"] = True - logger.setLevel(logging.DEBUG) - - -def is_verbose_mode_requested_in_pre_receive_scan() -> bool: - return does_git_push_option_have_value(VERBOSE_SCAN_FLAG) - - -def should_skip_pre_receive_scan() -> bool: - return does_git_push_option_have_value(SKIP_SCAN_FLAG) - - -def does_git_push_option_have_value(value: str) -> bool: - option_count_env_value = os.getenv(GIT_PUSH_OPTION_COUNT_ENV_VAR_NAME, '') - option_count = int(option_count_env_value) if option_count_env_value.isdigit() else 0 - return any(os.getenv(f'{GIT_PUSH_OPTION_ENV_VAR_PREFIX}{i}') == value for i in range(option_count)) diff --git a/cycode/cli/config.py b/cycode/cli/config.py index 9e023bfa..79a84fe2 100644 --- a/cycode/cli/config.py +++ b/cycode/cli/config.py @@ -1,14 +1,8 @@ -import os - -from cycode.cli.utils.yaml_utils import read_file from cycode.cli.user_settings.configuration_manager import ConfigurationManager - -relative_path = os.path.dirname(__file__) -config_file_path = os.path.join(relative_path, 'config.yaml') -config = read_file(config_file_path) configuration_manager = ConfigurationManager() # env vars CYCODE_CLIENT_ID_ENV_VAR_NAME = 'CYCODE_CLIENT_ID' CYCODE_CLIENT_SECRET_ENV_VAR_NAME = 'CYCODE_CLIENT_SECRET' +CYCODE_ID_TOKEN_ENV_VAR_NAME = 'CYCODE_ID_TOKEN' diff --git a/cycode/cli/config.yaml b/cycode/cli/config.yaml deleted file mode 100644 index 0ffe7abc..00000000 --- a/cycode/cli/config.yaml +++ /dev/null @@ -1,21 +0,0 @@ -soft_fail: False -scans: - supported_scans: - - secret - - iac - - sca - - sast - supported_sca_scans: - - package-vulnerabilities - - license-compliance -result_printer: - default: - lines_to_display: 3 - show_secret: False - secret: - pre_receive: - lines_to_display: 1 - show_secret: False - commit_history: - lines_to_display: 1 - show_secret: False \ No newline at end of file diff --git a/cycode/cli/console.py b/cycode/cli/console.py new file mode 100644 index 00000000..5d78fc36 --- /dev/null +++ b/cycode/cli/console.py @@ -0,0 +1,69 @@ +import os +from typing import TYPE_CHECKING, Optional + +from rich.console import Console, RenderResult +from rich.markdown import Heading, Markdown +from rich.text import Text + +if TYPE_CHECKING: + from rich.console import ConsoleOptions + +console_out = Console() +console_err = Console(stderr=True) + +console = console_out # alias + + +def is_dark_console() -> Optional[bool]: + """Detect if the console is dark or light. + + This function checks the environment variables and terminal type to determine if the console is dark or light. + + Used approaches: + 1. Check the `LC_DARK_BG` environment variable. + 2. Check the `COLORFGBG` environment variable for background color. + + And it still could be wrong in some cases. + + TODO(MarshalX): migrate to https://github.com/dalance/termbg when someone will implement it for Python. + """ + dark = None + + dark_bg = os.environ.get('LC_DARK_BG') + if dark_bg is not None: + return dark_bg != '0' + + # If BG color in {0, 1, 2, 3, 4, 5, 6, 8} then dark, else light. + try: + color = os.environ.get('COLORFGBG') + *_, bg = color.split(';') + bg = int(bg) + dark = bool(0 <= bg <= 6 or bg == 8) + except Exception: # noqa: S110 + pass + + return dark + + +_SYNTAX_HIGHLIGHT_DARK_THEME = 'monokai' +_SYNTAX_HIGHLIGHT_LIGHT_THEME = 'default' + +# when we could not detect it, use dark theme as most terminals are dark +_SYNTAX_HIGHLIGHT_THEME = _SYNTAX_HIGHLIGHT_LIGHT_THEME if is_dark_console() is False else _SYNTAX_HIGHLIGHT_DARK_THEME + + +class CycodeHeading(Heading): + """Custom Rich Heading for Markdown. + + Changes: + - remove justify to 'center' + - remove the box for h1 + """ + + def __rich_console__(self, console: 'Console', options: 'ConsoleOptions') -> RenderResult: + if self.tag == 'h2': + yield Text('') + yield self.text + + +Markdown.elements['heading_open'] = CycodeHeading diff --git a/cycode/cli/consts.py b/cycode/cli/consts.py index 3b0aaafe..31ab6ef9 100644 --- a/cycode/cli/consts.py +++ b/cycode/cli/consts.py @@ -1,71 +1,183 @@ -PRE_COMMIT_COMMAND_SCAN_TYPE = 'pre_commit' -PRE_RECEIVE_COMMAND_SCAN_TYPE = 'pre_receive' -COMMIT_HISTORY_COMMAND_SCAN_TYPE = 'commit_history' +PROGRAM_NAME = 'cycode' +APP_NAME = 'CycodeCLI' +CLI_CONTEXT_SETTINGS = {'terminal_width': 10**9, 'max_content_width': 10**9, 'help_option_names': ['-h', '--help']} + +PRE_COMMIT_COMMAND_SCAN_TYPE = 'pre-commit' +PRE_COMMIT_COMMAND_SCAN_TYPE_OLD = 'pre_commit' +PRE_RECEIVE_COMMAND_SCAN_TYPE = 'pre-receive' +PRE_RECEIVE_COMMAND_SCAN_TYPE_OLD = 'pre_receive' +COMMIT_HISTORY_COMMAND_SCAN_TYPE = 'commit-history' +COMMIT_HISTORY_COMMAND_SCAN_TYPE_OLD = 'commit_history' SECRET_SCAN_TYPE = 'secret' -INFRA_CONFIGURATION_SCAN_TYPE = 'iac' -SCA_SCAN_TYPE = "sca" -SAST_SCAN_TYPE = "sast" +IAC_SCAN_TYPE = 'iac' +SCA_SCAN_TYPE = 'sca' +SAST_SCAN_TYPE = 'sast' -INFRA_CONFIGURATION_SCAN_SUPPORTED_FILES = [ - '.tf', '.tf.json', '.json', '.yaml', '.yml', 'dockerfile' -] +IAC_SCAN_SUPPORTED_FILE_EXTENSIONS = ('.tf', '.tf.json', '.json', '.yaml', '.yml', '.dockerfile', '.containerfile') +IAC_SCAN_SUPPORTED_FILE_PREFIXES = ('dockerfile', 'containerfile') -SECRET_SCAN_FILE_EXTENSIONS_TO_IGNORE = [ - '.7z', '.bmp', '.bz2', '.dmg', '.exe', '.gif', '.gz', '.ico', '.jar', '.jpg', '.jpeg', '.png', '.rar', - '.realm', '.s7z', '.svg', '.tar', '.tif', '.tiff', '.webp', '.zi', '.lock', '.css', '.less', '.dll', - '.enc', '.deb', '.obj', '.model' -] +CYCODEIGNORE_FILENAME = '.cycodeignore' +CYCODE_ENTRYPOINT_FILENAME = 'entrypoint.cycode' -SCA_CONFIGURATION_SCAN_SUPPORTED_FILES = [ - 'cargo.lock', 'cargo.toml', - 'composer.json', 'composer.lock', - 'go.sum', 'go.mod', 'gopkg.lock', - 'pom.xml', 'build.gradle', 'gradle.lockfile', 'build.gradle.kts', - 'package.json', 'package-lock.json', 'yarn.lock', 'npm-shrinkwrap.json', - 'packages.config', 'project.assets.json', 'packages.lock.json', 'nuget.config', '.csproj', - 'gemfile', 'gemfile.lock', - 'build.sbt', 'build.scala', 'build.sbt.lock', - 'pyproject.toml', 'poetry.lock', - 'pipfile', 'pipfile.lock', 'requirements.txt', 'setup.py' -] +SECRET_SCAN_FILE_EXTENSIONS_TO_IGNORE = ( + '.DS_Store', + '.bmp', + '.gif', + '.ico', + '.tif', + '.tiff', + '.webp', + '.mp3', + '.mp4', + '.mkv', + '.avi', + '.mov', + '.mpg', + '.mpeg', + '.wav', + '.vob', + '.aac', + '.flac', + '.ogg', + '.mka', + '.wma', + '.wmv', + '.psd', + '.ai', + '.model', + '.lock', + '.css', + '.pdf', + '.odt', + '.iso', +) -SCA_EXCLUDED_PATHS = [ - 'node_modules' -] +SCA_CONFIGURATION_SCAN_SUPPORTED_FILES = ( # keep in lowercase + 'cargo.lock', + 'cargo.toml', + 'composer.json', + 'composer.lock', + 'go.sum', + 'go.mod', + 'go.mod.graph', + 'gopkg.lock', + 'pom.xml', + 'bom.json', + 'bcde.mvndeps', + 'build.gradle', + '.gradle', + 'gradle.lockfile', + 'build.gradle.kts', + '.gradle.kts', + '.properties', + '.kt', # config KT files + 'package.json', + 'package-lock.json', + 'yarn.lock', + 'deno.lock', + 'deno.json', + 'pnpm-lock.yaml', + 'npm-shrinkwrap.json', + 'packages.config', + 'project.assets.json', + 'packages.lock.json', + 'nuget.config', + '.csproj', + '.vbproj', + 'gemfile', + 'gemfile.lock', + '.sbt', + 'build.scala', + 'build.sbt.lock', + 'pyproject.toml', + 'poetry.lock', + 'pipfile', + 'pipfile.lock', + 'requirements.txt', + 'setup.py', + 'mix.exs', + 'mix.lock', + 'package.swift', + 'package.resolved', + 'pubspec.yaml', + 'pubspec.lock', + 'conanfile.py', + 'conanfile.txt', + 'maven_install.json', + 'conan.lock', +) + +SCA_EXCLUDED_FOLDER_IN_PATH = ( + 'node_modules', + 'venv', + '.venv', + '__pycache__', + '.pytest_cache', + '.tox', + '.mvn', + '.gradle', + '.npm', + '.yarn', + '.bundle', + '.bloop', + '.build', + '.dart_tool', + '.pub', +) PROJECT_FILES_BY_ECOSYSTEM_MAP = { - "crates": ["Cargo.lock", "Cargo.toml"], - "composer": ["composer.json", "composer.lock"], - "go": ["go.sum", "go.mod", "Gopkg.lock"], - "maven_pom": ["pom.xml"], - "maven_gradle": ["build.gradle", "build.gradle.kts", "gradle.lockfile"], - "npm": ["package.json", "package-lock.json", "yarn.lock", "npm-shrinkwrap.json", ".npmrc"], - "nuget": ["packages.config", "project.assets.json", "packages.lock.json", "nuget.config"], - "ruby_gems": ["Gemfile", "Gemfile.lock"], - "sbt": ["build.sbt", "build.scala", "build.sbt.lock"], - "pypi_poetry": ["pyproject.toml", "poetry.lock"], - "pypi_pipenv": ["Pipfile", "Pipfile.lock"], - "pypi_requirements": ["requirements.txt"], - "pypi_setup": ["setup.py"] + 'crates': ['Cargo.lock', 'Cargo.toml'], + 'composer': ['composer.json', 'composer.lock'], + 'go': ['go.sum', 'go.mod', 'go.mod.graph', 'Gopkg.lock'], + 'maven_pom': ['pom.xml'], + 'maven_gradle': ['build.gradle', 'build.gradle.kts', 'gradle.lockfile'], + 'npm': [ + 'package.json', + 'package-lock.json', + 'yarn.lock', + 'npm-shrinkwrap.json', + '.npmrc', + 'pnpm-lock.yaml', + 'deno.lock', + 'deno.json', + ], + 'nuget': ['packages.config', 'project.assets.json', 'packages.lock.json', 'nuget.config'], + 'ruby_gems': ['Gemfile', 'Gemfile.lock'], + 'sbt': ['build.sbt', 'build.scala', 'build.sbt.lock'], + 'pypi_poetry': ['pyproject.toml', 'poetry.lock'], + 'pypi_pipenv': ['Pipfile', 'Pipfile.lock'], + 'pypi_requirements': ['requirements.txt'], + 'pypi_setup': ['setup.py'], + 'hex': ['mix.exs', 'mix.lock'], + 'swift_pm': ['Package.swift', 'Package.resolved'], + 'dart': ['pubspec.yaml', 'pubspec.lock'], + 'conan': ['conanfile.py', 'conanfile.txt', 'conan.lock'], } -COMMIT_RANGE_SCAN_SUPPORTED_SCAN_TYPES = [SECRET_SCAN_TYPE, SCA_SCAN_TYPE] +COMMIT_RANGE_SCAN_SUPPORTED_SCAN_TYPES = [SECRET_SCAN_TYPE, SCA_SCAN_TYPE, SAST_SCAN_TYPE] -COMMIT_RANGE_BASED_COMMAND_SCAN_TYPES = [PRE_RECEIVE_COMMAND_SCAN_TYPE, COMMIT_HISTORY_COMMAND_SCAN_TYPE] +COMMIT_RANGE_BASED_COMMAND_SCAN_TYPES = [ + PRE_COMMIT_COMMAND_SCAN_TYPE, + PRE_COMMIT_COMMAND_SCAN_TYPE_OLD, + PRE_RECEIVE_COMMAND_SCAN_TYPE, + PRE_RECEIVE_COMMAND_SCAN_TYPE_OLD, + COMMIT_HISTORY_COMMAND_SCAN_TYPE, + COMMIT_HISTORY_COMMAND_SCAN_TYPE_OLD, +] -DEFAULT_CYCODE_API_URL = "https://api.cycode.com" -DEFAULT_CYCODE_APP_URL = "https://app.cycode.com" +DEFAULT_CYCODE_DOMAIN = 'cycode.com' +DEFAULT_CYCODE_API_URL = f'https://api.{DEFAULT_CYCODE_DOMAIN}' +DEFAULT_CYCODE_APP_URL = f'https://app.{DEFAULT_CYCODE_DOMAIN}' # env var names -CYCODE_API_URL_ENV_VAR_NAME = "CYCODE_API_URL" -CYCODE_APP_URL_ENV_VAR_NAME = "CYCODE_APP_URL" -TIMEOUT_ENV_VAR_NAME = "TIMEOUT" -CYCODE_CLI_REQUEST_TIMEOUT_ENV_VAR_NAME = "CYCODE_CLI_REQUEST_TIMEOUT" -LOGGING_LEVEL_ENV_VAR_NAME = "LOGGING_LEVEL" -# use only for dev envs locally -BATCH_SIZE_ENV_VAR_NAME = "BATCH_SIZE" -VERBOSE_ENV_VAR_NAME = "CYCODE_CLI_VERBOSE" +CYCODE_API_URL_ENV_VAR_NAME = 'CYCODE_API_URL' +CYCODE_APP_URL_ENV_VAR_NAME = 'CYCODE_APP_URL' +TIMEOUT_ENV_VAR_NAME = 'TIMEOUT' +CYCODE_CLI_REQUEST_TIMEOUT_ENV_VAR_NAME = 'CYCODE_CLI_REQUEST_TIMEOUT' +LOGGING_LEVEL_ENV_VAR_NAME = 'LOGGING_LEVEL' +VERBOSE_ENV_VAR_NAME = 'CYCODE_CLI_VERBOSE' +DEBUG_ENV_VAR_NAME = 'CYCODE_CLI_DEBUG' CYCODE_CONFIGURATION_DIRECTORY: str = '.cycode' @@ -75,48 +187,87 @@ EXCLUSIONS_BY_PATH_SECTION_NAME = 'paths' EXCLUSIONS_BY_RULE_SECTION_NAME = 'rules' EXCLUSIONS_BY_PACKAGE_SECTION_NAME = 'packages' +EXCLUSIONS_BY_CVE_SECTION_NAME = 'cves' + +# 5MB in bytes (in decimal) +FILE_MAX_SIZE_LIMIT_IN_BYTES = 5000000 + +PRESIGNED_LINK_UPLOADED_ZIP_MAX_SIZE_LIMIT_IN_BYTES = 5 * 1024 * 1024 * 1024 # 5 GB (S3 presigned POST limit) +PRESIGNED_UPLOAD_SCAN_TYPES = {SAST_SCAN_TYPE} + +DEFAULT_ZIP_MAX_SIZE_LIMIT_IN_BYTES = 20 * 1024 * 1024 +ZIP_MAX_SIZE_LIMIT_IN_BYTES = { + SCA_SCAN_TYPE: 200 * 1024 * 1024, + SAST_SCAN_TYPE: PRESIGNED_LINK_UPLOADED_ZIP_MAX_SIZE_LIMIT_IN_BYTES, +} + +# scan in batches +DEFAULT_SCAN_BATCH_MAX_SIZE_IN_BYTES = 9 * 1024 * 1024 +SCAN_BATCH_MAX_SIZE_IN_BYTES = {SAST_SCAN_TYPE: PRESIGNED_LINK_UPLOADED_ZIP_MAX_SIZE_LIMIT_IN_BYTES} +SCAN_BATCH_MAX_SIZE_IN_BYTES_ENV_VAR_NAME = 'SCAN_BATCH_MAX_SIZE_IN_BYTES' + +DEFAULT_SCAN_BATCH_MAX_FILES_COUNT = 1000 +SCAN_BATCH_MAX_FILES_COUNT_ENV_VAR_NAME = 'SCAN_BATCH_MAX_FILES_COUNT' + +# if we increase this values, the server doesn't allow connecting (ConnectionError) +SCAN_BATCH_MAX_PARALLEL_SCANS = 5 +SCAN_BATCH_SCANS_PER_CPU = 1 + +# sync scans +SYNC_SCAN_TIMEOUT_IN_SECONDS_ENV_VAR_NAME = 'SYNC_SCAN_TIMEOUT_IN_SECONDS' +DEFAULT_SYNC_SCAN_TIMEOUT_IN_SECONDS = 180 -# 1MB in bytes (in decimal) -FILE_MAX_SIZE_LIMIT_IN_BYTES = 1000000 +# ai remediation +AI_REMEDIATION_TIMEOUT_IN_SECONDS_ENV_VAR_NAME = 'AI_REMEDIATION_TIMEOUT_IN_SECONDS' +DEFAULT_AI_REMEDIATION_TIMEOUT_IN_SECONDS = 60 -# 20MB in bytes (in binary) -ZIP_MAX_SIZE_LIMIT_IN_BYTES = 20971520 -# 200MB in bytes (in binary) -SCA_ZIP_MAX_SIZE_LIMIT_IN_BYTES = 209715200 +# report with polling +REPORT_POLLING_WAIT_INTERVAL_IN_SECONDS = 5 +DEFAULT_REPORT_POLLING_TIMEOUT_IN_SECONDS = 600 +REPORT_POLLING_TIMEOUT_IN_SECONDS_ENV_VAR_NAME = 'REPORT_POLLING_TIMEOUT_IN_SECONDS' # scan with polling SCAN_POLLING_WAIT_INTERVAL_IN_SECONDS = 5 DEFAULT_SCAN_POLLING_TIMEOUT_IN_SECONDS = 3600 SCAN_POLLING_TIMEOUT_IN_SECONDS_ENV_VAR_NAME = 'SCAN_POLLING_TIMEOUT_IN_SECONDS' -DETECTIONS_COUNT_VERIFICATION_TIMEOUT_IN_SECONDS = 600 -DETECTIONS_COUNT_VERIFICATION_WAIT_INTERVAL_IN_SECONDS = 10 DEFAULT_SCA_PRE_COMMIT_TIMEOUT_IN_SECONDS = 600 SCA_PRE_COMMIT_TIMEOUT_IN_SECONDS_ENV_VAR_NAME = 'SCA_PRE_COMMIT_TIMEOUT_IN_SECONDS' -PROGRESS_UPDATE_COMMITS_INTERVAL = 100 - # pre receive scan PRE_RECEIVE_MAX_COMMITS_TO_SCAN_COUNT_ENV_VAR_NAME = 'PRE_RECEIVE_MAX_COMMITS_TO_SCAN_COUNT' DEFAULT_PRE_RECEIVE_MAX_COMMITS_TO_SCAN_COUNT = 50 PRE_RECEIVE_COMMAND_TIMEOUT_ENV_VAR_NAME = 'PRE_RECEIVE_COMMAND_TIMEOUT' DEFAULT_PRE_RECEIVE_COMMAND_TIMEOUT_IN_SECONDS = 60 -PRE_RECEIVE_REMEDIATION_MESSAGE = """ +# pre push scan +PRE_PUSH_MAX_COMMITS_TO_SCAN_COUNT_ENV_VAR_NAME = 'PRE_PUSH_MAX_COMMITS_TO_SCAN_COUNT' +DEFAULT_PRE_PUSH_MAX_COMMITS_TO_SCAN_COUNT = 50 +PRE_PUSH_COMMAND_TIMEOUT_ENV_VAR_NAME = 'PRE_PUSH_COMMAND_TIMEOUT' +DEFAULT_PRE_PUSH_COMMAND_TIMEOUT_IN_SECONDS = 60 +CYCODE_DEFAULT_BRANCH_ENV_VAR_NAME = 'CYCODE_DEFAULT_BRANCH' +# pre push and pre receive common +PRE_RECEIVE_AND_PUSH_REMEDIATION_MESSAGE = """ Cycode Secrets Push Protection ------------------------------------------------------------------------------ Resolve the following secrets by rewriting your local commit history before pushing again. -Learn how to: https://cycode.com/dont-let-hardcoded-secrets-compromise-your-security-4-effective-remediation-techniques +Learn how to: https://cycode.com/dont-let-hardcoded-secrets-compromise-your-security-4-effective-remediation-techniques """ EXCLUDE_DETECTIONS_IN_DELETED_LINES_ENV_VAR_NAME = 'EXCLUDE_DETECTIONS_IN_DELETED_LINES' DEFAULT_EXCLUDE_DETECTIONS_IN_DELETED_LINES = True +# report statuses +REPORT_STATUS_COMPLETED = 'Completed' +REPORT_STATUS_ERROR = 'Failed' + # scan statuses SCAN_STATUS_COMPLETED = 'Completed' SCAN_STATUS_ERROR = 'Error' # git consts COMMIT_DIFF_DELETED_FILE_CHANGE_TYPE = 'D' +COMMIT_RANGE_ALL_COMMITS = '--all' GIT_HEAD_COMMIT_REV = 'HEAD' +GIT_EMPTY_TREE_OBJECT = '4b825dc642cb6eb9a060e54bf8d69288fbee4904' EMPTY_COMMIT_SHA = '0000000000000000000000000000000000000000' GIT_PUSH_OPTION_COUNT_ENV_VAR_NAME = 'GIT_PUSH_OPTION_COUNT' GIT_PUSH_OPTION_ENV_VAR_PREFIX = 'GIT_PUSH_OPTION_' @@ -124,5 +275,18 @@ SKIP_SCAN_FLAG = 'skip-cycode-scan' VERBOSE_SCAN_FLAG = 'verbose' +ISSUE_DETECTED_STATUS_CODE = 1 +NO_ISSUES_STATUS_CODE = 0 + LICENSE_COMPLIANCE_POLICY_ID = '8f681450-49e1-4f7e-85b7-0c8fe84b3a35' PACKAGE_VULNERABILITY_POLICY_ID = '9369d10a-9ac0-48d3-9921-5de7fe9a37a7' + +# Shortcut dependency paths by remove all middle dependencies +# between direct dependency and influence/vulnerable dependency. +# Example: A -> B -> C +# Result: A -> ... -> C +SCA_SHORTCUT_DEPENDENCY_PATHS = 2 + +PLASTIC_VCS_DATA_SEPARATOR = ':::' +PLASTIC_VSC_CLI_TIMEOUT = 10 +PLASTIC_VCS_REMOTE_URI_PREFIX = 'plastic::' diff --git a/cycode/cli/exceptions/custom_exceptions.py b/cycode/cli/exceptions/custom_exceptions.py index d80cf26a..78781914 100644 --- a/cycode/cli/exceptions/custom_exceptions.py +++ b/cycode/cli/exceptions/custom_exceptions.py @@ -1,55 +1,109 @@ from requests import Response +from cycode.cli.models import CliError, CliErrors + class CycodeError(Exception): - """Base class for all custom exceptions""" + """Base class for all custom exceptions.""" + + def __str__(self) -> str: + class_name = self.__class__.__name__ + return f'{class_name} error occurred.' + + +class RequestError(CycodeError): ... + + +class RequestTimeoutError(RequestError): ... + + +class RequestConnectionError(RequestError): ... + + +class RequestSslError(RequestConnectionError): ... -class NetworkError(CycodeError): - def __init__(self, status_code: int, error_message: str, response: Response): +class RequestHttpError(RequestError): + def __init__(self, status_code: int, error_message: str, response: Response) -> None: self.status_code = status_code self.error_message = error_message self.response = response super().__init__(self.error_message) - def __str__(self): - return f'error occurred during the request. status code: {self.status_code}, error message: ' \ - f'{self.error_message}' + def __str__(self) -> str: + return f'HTTP error occurred during the request (code {self.status_code}). Message: {self.error_message}' class ScanAsyncError(CycodeError): - def __init__(self, error_message: str): + def __init__(self, error_message: str) -> None: self.error_message = error_message super().__init__(self.error_message) - def __str__(self): - return f'error occurred during the scan. error message: {self.error_message}' + def __str__(self) -> str: + return f'Async scan error occurred during the scan. Message: {self.error_message}' -class HttpUnauthorizedError(CycodeError): - def __init__(self, error_message: str, response: Response): - self.status_code = 401 - self.error_message = error_message - self.response = response - super().__init__(self.error_message) +class ReportAsyncError(CycodeError): + pass + + +class HttpUnauthorizedError(RequestHttpError): + def __init__(self, error_message: str, response: Response) -> None: + super().__init__(401, error_message, response) - def __str__(self): - return 'Http Unauthorized Error' + def __str__(self) -> str: + return f'HTTP unauthorized error occurred during the request. Message: {self.error_message}' class ZipTooLargeError(CycodeError): - def __init__(self, size_limit: int): + def __init__(self, size_limit: int) -> None: self.size_limit = size_limit super().__init__() - def __str__(self): + def __str__(self) -> str: return f'The size of zip to scan is too large, size limit: {self.size_limit}' class AuthProcessError(CycodeError): - def __init__(self, error_message: str): + def __init__(self, error_message: str) -> None: self.error_message = error_message super().__init__() - def __str__(self): - return f'Something went wrong during the authentication process, error message: {self.error_message}' + def __str__(self) -> str: + return f'Something went wrong during the authentication process. Message: {self.error_message}' + + +class TfplanKeyError(CycodeError): + def __init__(self, file_path: str) -> None: + self.file_path = file_path + super().__init__() + + def __str__(self) -> str: + return f'Error occurred while parsing terraform plan file. Path: {self.file_path}' + + +KNOWN_USER_FRIENDLY_REQUEST_ERRORS: CliErrors = { + RequestHttpError: CliError( + soft_fail=True, + code='cycode_error', + message='Cycode was unable to complete this scan. Please try again by executing the `cycode scan` command', + ), + RequestTimeoutError: CliError( + soft_fail=True, + code='timeout_error', + message='The request timed out. Please try again by executing the `cycode scan` command', + ), + HttpUnauthorizedError: CliError( + soft_fail=True, + code='auth_error', + message='Unable to authenticate to Cycode, your token is either invalid or has expired. ' + 'Please re-generate your token and reconfigure it by running the `cycode configure` command', + ), + RequestSslError: CliError( + soft_fail=True, + code='ssl_error', + message='An SSL error occurred when trying to connect to the Cycode API. ' + 'If you use an on-premises installation or a proxy that intercepts SSL traffic ' + 'you should use the CURL_CA_BUNDLE environment variable to specify path to a valid .pem or similar', + ), +} diff --git a/cycode/cli/exceptions/handle_ai_remediation_errors.py b/cycode/cli/exceptions/handle_ai_remediation_errors.py new file mode 100644 index 00000000..961acd62 --- /dev/null +++ b/cycode/cli/exceptions/handle_ai_remediation_errors.py @@ -0,0 +1,22 @@ +import typer + +from cycode.cli.exceptions.custom_exceptions import KNOWN_USER_FRIENDLY_REQUEST_ERRORS, RequestHttpError +from cycode.cli.exceptions.handle_errors import handle_errors +from cycode.cli.models import CliError, CliErrors + + +class AiRemediationNotFoundError(Exception): ... + + +def handle_ai_remediation_exception(ctx: typer.Context, err: Exception) -> None: + if isinstance(err, RequestHttpError) and err.status_code == 404: + err = AiRemediationNotFoundError() + + errors: CliErrors = { + **KNOWN_USER_FRIENDLY_REQUEST_ERRORS, + AiRemediationNotFoundError: CliError( + code='ai_remediation_not_found', + message='The AI remediation was not found. Please try different detection ID', + ), + } + handle_errors(ctx, err, errors) diff --git a/cycode/cli/exceptions/handle_auth_errors.py b/cycode/cli/exceptions/handle_auth_errors.py new file mode 100644 index 00000000..72e18c88 --- /dev/null +++ b/cycode/cli/exceptions/handle_auth_errors.py @@ -0,0 +1,18 @@ +import typer + +from cycode.cli.exceptions.custom_exceptions import ( + KNOWN_USER_FRIENDLY_REQUEST_ERRORS, + AuthProcessError, +) +from cycode.cli.exceptions.handle_errors import handle_errors +from cycode.cli.models import CliError, CliErrors + + +def handle_auth_exception(ctx: typer.Context, err: Exception) -> None: + errors: CliErrors = { + **KNOWN_USER_FRIENDLY_REQUEST_ERRORS, + AuthProcessError: CliError( + code='auth_error', message='Authentication failed. Please try again later using the command `cycode auth`' + ), + } + handle_errors(ctx, err, errors) diff --git a/cycode/cli/exceptions/handle_errors.py b/cycode/cli/exceptions/handle_errors.py new file mode 100644 index 00000000..ded1d88c --- /dev/null +++ b/cycode/cli/exceptions/handle_errors.py @@ -0,0 +1,35 @@ +from typing import Optional + +import click +import typer + +from cycode.cli.models import CliError, CliErrors + + +def handle_errors( + ctx: typer.Context, err: BaseException, cli_errors: CliErrors, *, return_exception: bool = False +) -> Optional['CliError']: + printer = ctx.obj.get('console_printer') + printer.print_exception(err) + + if type(err) in cli_errors: + error = cli_errors[type(err)].enrich(additional_message=str(err)) + + if error.soft_fail is True: + ctx.obj['soft_fail'] = True + + if return_exception: + return error + + printer.print_error(error) + return None + + if isinstance(err, click.ClickException): + raise err + + unknown_error = CliError(code='unknown_error', message=str(err)) + if return_exception: + return unknown_error + + printer.print_error(unknown_error) + raise typer.Exit(1) diff --git a/cycode/cli/exceptions/handle_report_sbom_errors.py b/cycode/cli/exceptions/handle_report_sbom_errors.py new file mode 100644 index 00000000..22707c8c --- /dev/null +++ b/cycode/cli/exceptions/handle_report_sbom_errors.py @@ -0,0 +1,23 @@ +import typer + +from cycode.cli.exceptions import custom_exceptions +from cycode.cli.exceptions.custom_exceptions import KNOWN_USER_FRIENDLY_REQUEST_ERRORS +from cycode.cli.exceptions.handle_errors import handle_errors +from cycode.cli.models import CliError, CliErrors + + +def handle_report_exception(ctx: typer.Context, err: Exception) -> None: + errors: CliErrors = { + **KNOWN_USER_FRIENDLY_REQUEST_ERRORS, + custom_exceptions.ScanAsyncError: CliError( + code='report_error', + message='Cycode was unable to complete this report. ' + 'Please try again by executing the `cycode report` command', + ), + custom_exceptions.ReportAsyncError: CliError( + code='report_error', + message='Cycode was unable to complete this report. ' + 'Please try again by executing the `cycode report` command', + ), + } + handle_errors(ctx, err, errors) diff --git a/cycode/cli/exceptions/handle_scan_errors.py b/cycode/cli/exceptions/handle_scan_errors.py new file mode 100644 index 00000000..229e0f02 --- /dev/null +++ b/cycode/cli/exceptions/handle_scan_errors.py @@ -0,0 +1,45 @@ +from typing import Optional + +import typer + +from cycode.cli.exceptions import custom_exceptions +from cycode.cli.exceptions.custom_exceptions import KNOWN_USER_FRIENDLY_REQUEST_ERRORS +from cycode.cli.exceptions.handle_errors import handle_errors +from cycode.cli.models import CliError, CliErrors +from cycode.cli.utils.git_proxy import git_proxy + + +def handle_scan_exception(ctx: typer.Context, err: Exception, *, return_exception: bool = False) -> Optional[CliError]: + ctx.obj['did_fail'] = True + + errors: CliErrors = { + **KNOWN_USER_FRIENDLY_REQUEST_ERRORS, + custom_exceptions.ScanAsyncError: CliError( + soft_fail=True, + code='scan_error', + message='Cycode was unable to complete this scan. Please try again by executing the `cycode scan` command', + ), + custom_exceptions.ZipTooLargeError: CliError( + soft_fail=True, + code='zip_too_large_error', + message='The path you attempted to scan exceeds the current maximum scanning size cap (10MB). ' + 'Please try ignoring irrelevant paths using the `cycode ignore --by-path` command ' + 'and execute the scan again', + ), + custom_exceptions.TfplanKeyError: CliError( + soft_fail=True, + code='key_error', + message=f'\n{err!s}\n' + 'A crucial field is missing in your terraform plan file. ' + 'Please make sure that your file is well formed ' + 'and execute the scan again', + ), + git_proxy.get_invalid_git_repository_error(): CliError( + soft_fail=False, + code='invalid_git_error', + message='The path you supplied does not correlate to a Git repository. ' + 'If you still wish to scan this path, use: `cycode scan path `', + ), + } + + return handle_errors(ctx, err, errors, return_exception=return_exception) diff --git a/cycode/cli/files_collector/__init__.py b/cycode/cli/files_collector/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cycode/cli/files_collector/commit_range_documents.py b/cycode/cli/files_collector/commit_range_documents.py new file mode 100644 index 00000000..a4a1a784 --- /dev/null +++ b/cycode/cli/files_collector/commit_range_documents.py @@ -0,0 +1,515 @@ +import os +import sys +from typing import TYPE_CHECKING, Optional + +import typer + +from cycode.cli import consts +from cycode.cli.files_collector.repository_documents import ( + get_file_content_from_commit_path, +) +from cycode.cli.models import Document +from cycode.cli.utils.git_proxy import git_proxy +from cycode.cli.utils.path_utils import get_file_content, get_path_by_os +from cycode.cli.utils.progress_bar import ScanProgressBarSection +from cycode.logger import get_logger + +if TYPE_CHECKING: + from git import Diff, Repo + + from cycode.cli.utils.progress_bar import BaseProgressBar, ProgressBarSection + +logger = get_logger('Commit Range Collector') + + +def get_safe_head_reference_for_diff(repo: 'Repo') -> str: + """Get a safe reference to use for diffing against the current HEAD. + In repositories with no commits, HEAD doesn't exist, so we return the empty tree hash. + + Args: + repo: Git repository object + + Returns: + Either "HEAD" string if commits exist, or empty tree hash if no commits exist + """ + try: + repo.rev_parse(consts.GIT_HEAD_COMMIT_REV) + return consts.GIT_HEAD_COMMIT_REV + except Exception as e: # actually gitdb.exc.BadObject; no import because of lazy loading + logger.debug( + 'Repository has no commits, using empty tree hash for diffs, %s', + {'repo_path': repo.working_tree_dir}, + exc_info=e, + ) + + # Repository has no commits, use the universal empty tree hash + # This is the standard Git approach for initial commits + return consts.GIT_EMPTY_TREE_OBJECT + + +def _does_reach_to_max_commits_to_scan_limit(commit_ids: list[str], max_commits_count: Optional[int]) -> bool: + if max_commits_count is None: + return False + + return len(commit_ids) >= max_commits_count + + +def collect_commit_range_diff_documents( + ctx: typer.Context, path: str, commit_range: str, max_commits_count: Optional[int] = None +) -> list[Document]: + """Collects documents from a specified commit range in a Git repository. + + Return a list of Document objects containing the diffs of files changed in each commit. + """ + progress_bar = ctx.obj['progress_bar'] + + commit_ids_to_scan = [] + commit_documents_to_scan = [] + + repo = git_proxy.get_repo(path) + + normalized_commit_range = normalize_commit_range(commit_range, path) + + total_commits_count = int(repo.git.rev_list('--count', normalized_commit_range)) + logger.debug( + 'Calculating diffs for %s commits in the commit range %s', total_commits_count, normalized_commit_range + ) + + progress_bar.set_section_length(ScanProgressBarSection.PREPARE_LOCAL_FILES, total_commits_count) + + for scanned_commits_count, commit in enumerate(repo.iter_commits(rev=normalized_commit_range)): + if _does_reach_to_max_commits_to_scan_limit(commit_ids_to_scan, max_commits_count): + logger.debug('Reached to max commits to scan count. Going to scan only %s last commits', max_commits_count) + progress_bar.update(ScanProgressBarSection.PREPARE_LOCAL_FILES, total_commits_count - scanned_commits_count) + break + + progress_bar.update(ScanProgressBarSection.PREPARE_LOCAL_FILES) + + commit_id = commit.hexsha + commit_ids_to_scan.append(commit_id) + parent = commit.parents[0] if commit.parents else git_proxy.get_null_tree() + diff_index = commit.diff(parent, create_patch=True, R=True) + for diff in diff_index: + commit_documents_to_scan.append( + Document( + path=get_path_by_os(get_diff_file_path(diff, repo=repo)), + content=get_diff_file_content(diff), + is_git_diff_format=True, + unique_id=commit_id, + ) + ) + + logger.debug( + 'Found all relevant files in commit %s', + { + 'path': path, + 'commit_range': commit_range, + 'normalized_commit_range': normalized_commit_range, + 'commit_id': commit_id, + }, + ) + + logger.debug('List of commit ids to scan, %s', {'commit_ids': commit_ids_to_scan}) + + return commit_documents_to_scan + + +def calculate_pre_receive_commit_range(repo_path: str, branch_update_details: str) -> Optional[str]: + end_commit = _get_end_commit_from_branch_update_details(branch_update_details) + + # branch is deleted, no need to perform scan + if end_commit == consts.EMPTY_COMMIT_SHA: + return None + + repo = git_proxy.get_repo(repo_path) + start_commit = _get_oldest_unupdated_commit_for_branch(repo, end_commit) + + # no new commit to update found + if not start_commit: + return None + + # If the oldest not-yet-updated commit has no parent (root commit or orphaned history), + # using '~1' will fail. In that case, scan from the end commit, which effectively + # includes the entire history reachable from it (which is exactly what we need here). + + if not bool(repo.commit(start_commit).parents): + return f'{end_commit}' + + return f'{start_commit}~1...{end_commit}' + + +def _get_end_commit_from_branch_update_details(update_details: str) -> str: + # update details pattern: + _, end_commit, _ = update_details.split() + return end_commit + + +def _get_oldest_unupdated_commit_for_branch(repo: 'Repo', commit: str) -> Optional[str]: + # get a list of commits by chronological order that are not in the remote repository yet + # more info about rev-list command: https://git-scm.com/docs/git-rev-list + + not_updated_commits = repo.git.rev_list(commit, '--topo-order', '--reverse', '--not', '--all') + + commits = not_updated_commits.splitlines() + if not commits: + return None + + return commits[0] + + +def _get_file_content_from_commit_diff(repo: 'Repo', commit: str, diff: 'Diff') -> Optional[str]: + file_path = get_diff_file_path(diff, relative=True) + return get_file_content_from_commit_path(repo, commit, file_path) + + +def get_commit_range_modified_documents( + progress_bar: 'BaseProgressBar', + progress_bar_section: 'ProgressBarSection', + path: str, + from_commit_rev: str, + to_commit_rev: str, + reverse_diff: bool = True, +) -> tuple[list[Document], list[Document], list[Document]]: + from_commit_documents = [] + to_commit_documents = [] + diff_documents = [] + + repo = git_proxy.get_repo(path) + diff_index = repo.commit(from_commit_rev).diff(to_commit_rev, create_patch=True, R=reverse_diff) + + modified_files_diff = [ + diff for diff in diff_index if diff.change_type != consts.COMMIT_DIFF_DELETED_FILE_CHANGE_TYPE + ] + progress_bar.set_section_length(progress_bar_section, len(modified_files_diff)) + for diff in modified_files_diff: + progress_bar.update(progress_bar_section) + + file_path = get_path_by_os(get_diff_file_path(diff, repo=repo)) + + diff_documents.append( + Document( + path=file_path, + content=get_diff_file_content(diff), + is_git_diff_format=True, + ) + ) + + file_content = _get_file_content_from_commit_diff(repo, from_commit_rev, diff) + if file_content is not None: + from_commit_documents.append(Document(file_path, file_content)) + + file_content = _get_file_content_from_commit_diff(repo, to_commit_rev, diff) + if file_content is not None: + to_commit_documents.append(Document(file_path, file_content)) + + return from_commit_documents, to_commit_documents, diff_documents + + +def parse_pre_receive_input() -> str: + """Parse input to pushed branch update details. + + Example input: + old_value new_value refname + ----------------------------------------------- + 0000000000000000000000000000000000000000 9cf90954ef26e7c58284f8ebf7dcd0fcf711152a refs/heads/main + 973a96d3e925b65941f7c47fa16129f1577d499f 0000000000000000000000000000000000000000 refs/heads/feature-branch + 59564ef68745bca38c42fc57a7822efd519a6bd9 3378e52dcfa47fb11ce3a4a520bea5f85d5d0bf3 refs/heads/develop + + :return: First branch update details (input's first line) + """ + pre_receive_input = _read_hook_input_from_stdin() + if not pre_receive_input: + raise ValueError( + 'Pre receive input was not found. Make sure that you are using this command only in pre-receive hook' + ) + + # each line represents a branch update request, handle the first one only + # TODO(MichalBor): support case of multiple update branch requests + return pre_receive_input.splitlines()[0] + + +def parse_pre_push_input() -> str: + """Parse input to pre-push hook details. + + Example input: + local_ref local_object_name remote_ref remote_object_name + --------------------------------------------------------- + refs/heads/main 9cf90954ef26e7c58284f8ebf7dcd0fcf711152a refs/heads/main 973a96d3e925b65941f7c47fa16129f1577d499f + refs/heads/feature-branch 3378e52dcfa47fb11ce3a4a520bea5f85d5d0bf3 refs/heads/feature-branch 59564ef68745bca38c42fc57a7822efd519a6bd9 + + :return: First, push update details (input's first line) + """ # noqa: E501 + pre_push_input = _read_hook_input_from_stdin() + if not pre_push_input: + raise ValueError( + 'Pre push input was not found. Make sure that you are using this command only in pre-push hook' + ) + + # each line represents a branch push request, handle the first one only + return pre_push_input.splitlines()[0] + + +def _read_hook_input_from_stdin() -> str: + """Read input from stdin when called from a hook. + + If called manually from the command line, return an empty string so it doesn't block the main thread. + + Returns: + Input from stdin + """ + if sys.stdin.isatty(): + return '' + return sys.stdin.read().strip() + + +def _get_default_branches_for_merge_base(repo: 'Repo') -> list[str]: + """Get a list of default branches to try for merge base calculation. + + Priority order: + 1. Environment variable CYCODE_DEFAULT_BRANCH + 2. Git remote HEAD (git symbolic-ref refs/remotes/origin/HEAD) + 3. Fallback to common default branch names + + Args: + repo: Git repository object + + Returns: + List of branch names to try for merge base calculation + """ + default_branches = [] + + # 1. Check environment variable first + env_default_branch = os.getenv(consts.CYCODE_DEFAULT_BRANCH_ENV_VAR_NAME) + if env_default_branch: + logger.debug('Using default branch from environment variable: %s', env_default_branch) + default_branches.append(env_default_branch) + + # 2. Try to get the actual default branch from remote HEAD + try: + remote_head = repo.git.symbolic_ref('refs/remotes/origin/HEAD') + # symbolic-ref returns something like "refs/remotes/origin/main" + if remote_head.startswith('refs/remotes/origin/'): + default_branch = remote_head.replace('refs/remotes/origin/', '') + logger.debug('Found remote default branch: %s', default_branch) + # Add both the remote tracking branch and local branch variants + default_branches.extend([f'origin/{default_branch}', default_branch]) + except Exception as e: + logger.debug('Failed to get remote HEAD via symbolic-ref: %s', exc_info=e) + + # Try an alternative method: git remote show origin + try: + remote_info = repo.git.remote('show', 'origin') + for line in remote_info.splitlines(): + if 'HEAD branch:' in line: + default_branch = line.split('HEAD branch:')[1].strip() + logger.debug('Found default branch via remote show: %s', default_branch) + default_branches.extend([f'origin/{default_branch}', default_branch]) + break + except Exception as e2: + logger.debug('Failed to get remote info via remote show: %s', exc_info=e2) + + # 3. Add fallback branches (avoiding duplicates) + fallback_branches = ['origin/main', 'origin/master', 'main', 'master'] + for branch in fallback_branches: + if branch not in default_branches: + default_branches.append(branch) + + logger.debug('Default branches to try: %s', default_branches) + return default_branches + + +def calculate_pre_push_commit_range(push_update_details: str) -> Optional[str]: + """Calculate the commit range for pre-push hook scanning. + + Args: + push_update_details: String in format "local_ref local_object_name remote_ref remote_object_name" + + Returns: + Commit range string for scanning, or None if no scanning is needed + + Environment Variables: + CYCODE_DEFAULT_BRANCH: Override the default branch for merge base calculation + """ + local_ref, local_object_name, remote_ref, remote_object_name = push_update_details.split() + + if remote_object_name == consts.EMPTY_COMMIT_SHA: + try: + repo = git_proxy.get_repo(os.getcwd()) + default_branches = _get_default_branches_for_merge_base(repo) + + merge_base = None + for default_branch in default_branches: + try: + merge_base = repo.git.merge_base(local_object_name, default_branch) + logger.debug('Found merge base %s with branch %s', merge_base, default_branch) + break + except Exception as e: + logger.debug('Failed to find merge base with %s: %s', default_branch, exc_info=e) + continue + + if merge_base: + return f'{merge_base}..{local_object_name}' + + logger.debug('Failed to find merge base with any default branch') + return consts.COMMIT_RANGE_ALL_COMMITS + except Exception as e: + logger.debug('Failed to get repo for pre-push commit range calculation: %s', exc_info=e) + return consts.COMMIT_RANGE_ALL_COMMITS + + # If deleting a branch (local_object_name is all zeros), no need to scan + if local_object_name == consts.EMPTY_COMMIT_SHA: + return None + + # For updates to existing branches, scan from remote to local + return f'{remote_object_name}..{local_object_name}' + + +def get_diff_file_path(diff: 'Diff', relative: bool = False, repo: Optional['Repo'] = None) -> Optional[str]: + """Get the file path from a git Diff object. + + Args: + diff: Git Diff object representing the file change + relative: If True, return the path relative to the repository root; + otherwise, return an absolute path IF possible + repo: Optional Git Repo object, used to resolve absolute paths + + Note: + It tries to get the absolute path, falling back to relative paths. `relative` flag forces relative paths. + + One case of relative paths is when the repository is bare and does not have a working tree directory. + """ + # try blob-based paths first (most reliable when available) + blob = diff.b_blob if diff.b_blob else diff.a_blob + if blob: + if relative: + return blob.path + if repo and repo.working_tree_dir: + return blob.abspath + + path = diff.b_path if diff.b_path else diff.a_path # relative path within the repo + if not relative and path and repo and repo.working_tree_dir: + # convert to the absolute path using the repo's working tree directory + path = os.path.join(repo.working_tree_dir, path) + + return path + + +def get_diff_file_content(diff: 'Diff') -> str: + return diff.diff.decode('UTF-8', errors='replace') + + +def get_pre_commit_modified_documents( + progress_bar: 'BaseProgressBar', + progress_bar_section: 'ProgressBarSection', + repo_path: str, +) -> tuple[list[Document], list[Document], list[Document]]: + git_head_documents = [] + pre_committed_documents = [] + diff_documents = [] + + repo = git_proxy.get_repo(repo_path) + head_reference = get_safe_head_reference_for_diff(repo) + diff_index = repo.index.diff(head_reference, create_patch=True, R=True) + progress_bar.set_section_length(progress_bar_section, len(diff_index)) + for diff in diff_index: + progress_bar.update(progress_bar_section) + + file_path = get_path_by_os(get_diff_file_path(diff, repo=repo)) + + diff_documents.append( + Document( + path=file_path, + content=get_diff_file_content(diff), + is_git_diff_format=True, + ) + ) + + # Only get file content from HEAD if HEAD exists (not the empty tree hash) + if head_reference == consts.GIT_HEAD_COMMIT_REV: + file_content = _get_file_content_from_commit_diff(repo, head_reference, diff) + if file_content: + git_head_documents.append(Document(file_path, file_content)) + + if os.path.exists(file_path): + file_content = get_file_content(file_path) + if file_content: + pre_committed_documents.append(Document(file_path, file_content)) + + return git_head_documents, pre_committed_documents, diff_documents + + +def parse_commit_range(commit_range: str, path: str) -> tuple[Optional[str], Optional[str], Optional[str]]: + """Parses a git commit range string and returns the full SHAs for the 'from' and 'to' commits. + Also, it returns the separator in the commit range. + + Supports: + - 'from..to' + - 'from...to' + - 'commit' (interpreted as 'commit..HEAD') + - '..to' (interpreted as 'HEAD..to') + - 'from..' (interpreted as 'from..HEAD') + - '--all' (interpreted as 'first_commit..HEAD' to scan all commits) + """ + repo = git_proxy.get_repo(path) + + # Handle '--all' special case: scan all commits from first to HEAD + # Usually represents an empty remote repository + if commit_range == consts.COMMIT_RANGE_ALL_COMMITS: + try: + head_commit = repo.rev_parse(consts.GIT_HEAD_COMMIT_REV).hexsha + all_commits = repo.git.rev_list('--reverse', head_commit).strip() + if all_commits: + first_commit = all_commits.splitlines()[0] + return first_commit, head_commit, '..' + logger.warning("No commits found for range '%s'", commit_range) + return None, None, None + except Exception as e: + logger.warning("Failed to parse commit range '%s'", commit_range, exc_info=e) + return None, None, None + + separator = '..' + if '...' in commit_range: + from_spec, to_spec = commit_range.split('...', 1) + separator = '...' + elif '..' in commit_range: + from_spec, to_spec = commit_range.split('..', 1) + else: + # Git commands like 'git diff ' compare against HEAD. + from_spec = commit_range + to_spec = consts.GIT_HEAD_COMMIT_REV + + # If a spec is empty (e.g., from '..master'), default it to 'HEAD' + if not from_spec: + from_spec = consts.GIT_HEAD_COMMIT_REV + if not to_spec: + to_spec = consts.GIT_HEAD_COMMIT_REV + + try: + # Use rev_parse to resolve each specifier to its full commit SHA + from_commit_rev = repo.rev_parse(from_spec).hexsha + to_commit_rev = repo.rev_parse(to_spec).hexsha + return from_commit_rev, to_commit_rev, separator + except git_proxy.get_git_command_error() as e: + logger.warning("Failed to parse commit range '%s'", commit_range, exc_info=e) + return None, None, None + + +def normalize_commit_range(commit_range: str, path: str) -> str: + """Normalize a commit range string to handle various formats consistently with all scan types. + + Returns: + A normalized commit range string suitable for Git operations (e.g., 'full_sha1..full_sha2') + """ + from_commit_rev, to_commit_rev, separator = parse_commit_range(commit_range, path) + if from_commit_rev is None or to_commit_rev is None: + logger.warning('Failed to parse commit range "%s", falling back to raw string.', commit_range) + return commit_range + + # Construct a normalized range string using the original separator for iter_commits + normalized_commit_range = f'{from_commit_rev}{separator}{to_commit_rev}' + logger.debug( + 'Normalized commit range "%s" to "%s"', + commit_range, + normalized_commit_range, + ) + return normalized_commit_range diff --git a/cycode/cli/files_collector/documents_walk_ignore.py b/cycode/cli/files_collector/documents_walk_ignore.py new file mode 100644 index 00000000..5a4dbe6d --- /dev/null +++ b/cycode/cli/files_collector/documents_walk_ignore.py @@ -0,0 +1,124 @@ +import os +from typing import TYPE_CHECKING + +from cycode.cli import consts +from cycode.cli.logger import get_logger +from cycode.cli.utils.ignore_utils import IgnoreFilterManager + +if TYPE_CHECKING: + from cycode.cli.models import Document + +logger = get_logger('Documents Ignores') + + +def _get_cycodeignore_path(repo_path: str) -> str: + """Get the path to .cycodeignore file in the repository root.""" + return os.path.join(repo_path, consts.CYCODEIGNORE_FILENAME) + + +def _create_ignore_filter_manager(repo_path: str, cycodeignore_path: str) -> IgnoreFilterManager: + """Create IgnoreFilterManager with .cycodeignore file.""" + return IgnoreFilterManager.build( + path=repo_path, + global_ignore_file_paths=[cycodeignore_path], + global_patterns=[], + ) + + +def _log_ignored_files(repo_path: str, dirpath: str, ignored_dirnames: list[str], ignored_filenames: list[str]) -> None: + """Log ignored files for debugging (similar to walk_ignore function).""" + rel_dirpath = '' if dirpath == repo_path else os.path.relpath(dirpath, repo_path) + display_dir = rel_dirpath or '.' + + for is_dir, names in ( + (True, ignored_dirnames), + (False, ignored_filenames), + ): + for name in names: + full_path = os.path.join(repo_path, display_dir, name) + if is_dir: + full_path = os.path.join(full_path, '*') + logger.debug('Ignoring match %s', full_path) + + +def _build_allowed_paths_set(ignore_filter_manager: IgnoreFilterManager, repo_path: str) -> set[str]: + """Build set of allowed file paths using walk_with_ignored.""" + allowed_paths = set() + + for dirpath, _dirnames, filenames, ignored_dirnames, ignored_filenames in ignore_filter_manager.walk_with_ignored(): + _log_ignored_files(repo_path, dirpath, ignored_dirnames, ignored_filenames) + + for filename in filenames: + file_path = os.path.join(dirpath, filename) + allowed_paths.add(file_path) + + return allowed_paths + + +def _get_document_check_path(document: 'Document', repo_path: str) -> str: + """Get the normalized absolute path for a document to check against allowed paths.""" + check_path = document.absolute_path + if not check_path: + check_path = document.path if os.path.isabs(document.path) else os.path.join(repo_path, document.path) + + return os.path.normpath(check_path) + + +def _filter_documents_by_allowed_paths( + documents: list['Document'], allowed_paths: set[str], repo_path: str +) -> list['Document']: + """Filter documents by checking if their paths are in the allowed set.""" + filtered_documents = [] + + for document in documents: + try: + check_path = _get_document_check_path(document, repo_path) + + if check_path in allowed_paths: + filtered_documents.append(document) + else: + relative_path = os.path.relpath(check_path, repo_path) + logger.debug('Filtered out document due to .cycodeignore: %s', relative_path) + except Exception as e: + logger.debug('Error processing document %s: %s', document.path, e) + filtered_documents.append(document) + + return filtered_documents + + +def filter_documents_with_cycodeignore( + documents: list['Document'], repo_path: str, is_cycodeignore_allowed: bool = True +) -> list['Document']: + """Filter documents based on .cycodeignore patterns. + + This function uses .cycodeignore file in the repository root to filter out + documents whose paths match any of those patterns. + + Args: + documents: List of Document objects to filter + repo_path: Path to the repository root + is_cycodeignore_allowed: Whether .cycodeignore filtering is allowed by scan configuration + + Returns: + List of Document objects that don't match any .cycodeignore patterns + """ + if not is_cycodeignore_allowed: + logger.debug('.cycodeignore filtering is not allowed by scan configuration') + return documents + + cycodeignore_path = _get_cycodeignore_path(repo_path) + + if not os.path.exists(cycodeignore_path): + logger.debug('.cycodeignore file does not exist in the repository root') + return documents + + logger.info('Using %s for filtering documents', cycodeignore_path) + + ignore_filter_manager = _create_ignore_filter_manager(repo_path, cycodeignore_path) + + allowed_paths = _build_allowed_paths_set(ignore_filter_manager, repo_path) + + filtered_documents = _filter_documents_by_allowed_paths(documents, allowed_paths, repo_path) + + logger.debug('Filtered %d documents using .cycodeignore patterns', len(documents) - len(filtered_documents)) + return filtered_documents diff --git a/cycode/cli/files_collector/file_excluder.py b/cycode/cli/files_collector/file_excluder.py new file mode 100644 index 00000000..11fd3410 --- /dev/null +++ b/cycode/cli/files_collector/file_excluder.py @@ -0,0 +1,191 @@ +from pathlib import Path +from typing import TYPE_CHECKING + +from cycode.cli import consts +from cycode.cli.config import configuration_manager +from cycode.cli.user_settings.config_file_manager import ConfigFileManager +from cycode.cli.utils.path_utils import get_file_size, is_binary_file, is_sub_path +from cycode.cli.utils.string_utils import get_content_size, is_binary_content +from cycode.logger import get_logger + +if TYPE_CHECKING: + from cycode.cli.models import Document + from cycode.cli.utils.progress_bar import BaseProgressBar, ProgressBarSection + from cycode.cyclient import models + + +logger = get_logger('File Excluder') + + +def _is_subpath_of_cycode_configuration_folder(filename: str) -> bool: + return ( + is_sub_path(configuration_manager.global_config_file_manager.get_config_directory_path(), filename) + or is_sub_path(configuration_manager.local_config_file_manager.get_config_directory_path(), filename) + or filename.endswith(ConfigFileManager.get_config_file_route()) + ) + + +def _is_path_configured_in_exclusions(scan_type: str, file_path: str) -> bool: + exclusions_by_path = configuration_manager.get_exclusions_by_scan_type(scan_type).get( + consts.EXCLUSIONS_BY_PATH_SECTION_NAME, [] + ) + return any(is_sub_path(exclusion_path, file_path) for exclusion_path in exclusions_by_path) + + +def _does_file_exceed_max_size_limit(filename: str) -> bool: + return get_file_size(filename) > consts.FILE_MAX_SIZE_LIMIT_IN_BYTES + + +def _does_document_exceed_max_size_limit(content: str) -> bool: + return get_content_size(content) > consts.FILE_MAX_SIZE_LIMIT_IN_BYTES + + +def _is_file_relevant_for_sca_scan(filename: str) -> bool: + for part in Path(filename).parts: + if part in consts.SCA_EXCLUDED_FOLDER_IN_PATH: + logger.debug( + 'The file is irrelevant because it is from an excluded directory, %s', + {'filename': filename, 'excluded_directory': part}, + ) + return False + + return True + + +class Excluder: + def __init__(self) -> None: + self._scannable_prefixes: dict[str, tuple[str, ...]] = { + consts.IAC_SCAN_TYPE: consts.IAC_SCAN_SUPPORTED_FILE_PREFIXES, + } + self._scannable_extensions: dict[str, tuple[str, ...]] = { + consts.IAC_SCAN_TYPE: consts.IAC_SCAN_SUPPORTED_FILE_EXTENSIONS, + consts.SCA_SCAN_TYPE: consts.SCA_CONFIGURATION_SCAN_SUPPORTED_FILES, + } + self._non_scannable_extensions: dict[str, tuple[str, ...]] = { + consts.SECRET_SCAN_TYPE: consts.SECRET_SCAN_FILE_EXTENSIONS_TO_IGNORE, + } + + def apply_scan_config(self, scan_type: str, scan_config: 'models.ScanConfiguration') -> None: + if scan_config.scannable_extensions: + self._scannable_extensions[scan_type] = tuple(scan_config.scannable_extensions) + + def _is_file_prefix_supported(self, scan_type: str, file_path: str) -> bool: + scannable_prefixes = self._scannable_prefixes.get(scan_type) + if scannable_prefixes: + path = Path(file_path) + file_name = path.name.lower() + return file_name in scannable_prefixes + return False + + def _is_file_extension_supported(self, scan_type: str, filename: str) -> bool: + filename = filename.lower() + + scannable_extensions = self._scannable_extensions.get(scan_type) + if scannable_extensions: + return filename.endswith(scannable_extensions) + + non_scannable_extensions = self._non_scannable_extensions.get(scan_type) + if non_scannable_extensions: + return not filename.endswith(non_scannable_extensions) + + return True + + def _is_relevant_file_to_scan_common(self, scan_type: str, filename: str) -> bool: + if _is_subpath_of_cycode_configuration_folder(filename): + logger.debug( + 'The document is irrelevant because it is in the Cycode configuration directory, %s', + {'filename': filename, 'configuration_directory': consts.CYCODE_CONFIGURATION_DIRECTORY}, + ) + return False + + if _is_path_configured_in_exclusions(scan_type, filename): + logger.debug( + 'The document is irrelevant because its path is in the ignore paths list, %s', {'filename': filename} + ) + return False + + if not ( + self._is_file_extension_supported(scan_type, filename) + or self._is_file_prefix_supported(scan_type, filename) + ): + logger.debug( + 'The document is irrelevant because its extension is not supported, %s', + {'scan_type': scan_type, 'filename': filename}, + ) + return False + + return True + + def _is_relevant_file_to_scan(self, scan_type: str, filename: str) -> bool: + if not self._is_relevant_file_to_scan_common(scan_type, filename): + return False + + if is_binary_file(filename): + logger.debug('The file is irrelevant because it is a binary file, %s', {'filename': filename}) + return False + + if scan_type != consts.SCA_SCAN_TYPE and _does_file_exceed_max_size_limit(filename): + logger.debug( + 'The file is irrelevant because it has exceeded the maximum size limit, %s', + { + 'max_file_size': consts.FILE_MAX_SIZE_LIMIT_IN_BYTES, + 'file_size': get_file_size(filename), + 'filename': filename, + }, + ) + return False + + return not (scan_type == consts.SCA_SCAN_TYPE and not _is_file_relevant_for_sca_scan(filename)) + + def _is_relevant_document_to_scan(self, scan_type: str, filename: str, content: str) -> bool: + if not self._is_relevant_file_to_scan_common(scan_type, filename): + return False + + if is_binary_content(content): + logger.debug('The document is irrelevant because it is a binary file, %s', {'filename': filename}) + return False + + if scan_type != consts.SCA_SCAN_TYPE and _does_document_exceed_max_size_limit(content): + logger.debug( + 'The document is irrelevant because it has exceeded the maximum size limit, %s', + { + 'max_document_size': consts.FILE_MAX_SIZE_LIMIT_IN_BYTES, + 'document_size': get_content_size(content), + 'filename': filename, + }, + ) + return False + + return True + + def exclude_irrelevant_files( + self, + progress_bar: 'BaseProgressBar', + progress_bar_section: 'ProgressBarSection', + scan_type: str, + filenames: list[str], + ) -> list[str]: + relevant_files = [] + for filename in filenames: + progress_bar.update(progress_bar_section) + if self._is_relevant_file_to_scan(scan_type, filename): + relevant_files.append(filename) + + is_sub_path.cache_clear() # free up memory + + return relevant_files + + def exclude_irrelevant_documents_to_scan( + self, scan_type: str, documents_to_scan: list['Document'] + ) -> list['Document']: + logger.debug('Excluding irrelevant documents to scan') + + relevant_documents = [] + for document in documents_to_scan: + if self._is_relevant_document_to_scan(scan_type, document.path, document.content): + relevant_documents.append(document) + + return relevant_documents + + +excluder = Excluder() diff --git a/cycode/cli/files_collector/iac/__init__.py b/cycode/cli/files_collector/iac/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cycode/cli/files_collector/iac/tf_content_generator.py b/cycode/cli/files_collector/iac/tf_content_generator.py new file mode 100644 index 00000000..63be9e47 --- /dev/null +++ b/cycode/cli/files_collector/iac/tf_content_generator.py @@ -0,0 +1,82 @@ +import json +import time + +from cycode.cli import consts +from cycode.cli.exceptions.custom_exceptions import TfplanKeyError +from cycode.cli.models import ResourceChange +from cycode.cli.utils.path_utils import change_filename_extension, load_json + +ACTIONS_TO_OMIT_RESOURCE = ['delete'] + + +def generate_tfplan_document_name(path: str) -> str: + document_name = change_filename_extension(path, 'tf') + timestamp = int(time.time()) + return f'{timestamp}-{document_name}' + + +def is_iac(scan_type: str) -> bool: + return scan_type == consts.IAC_SCAN_TYPE + + +def is_tfplan_file(file: str, content: str) -> bool: + if not file.endswith('.json'): + return False + tf_plan = load_json(content) + if not isinstance(tf_plan, dict): + return False + return 'resource_changes' in tf_plan + + +def generate_tf_content_from_tfplan(filename: str, tfplan: str) -> str: + planned_resources = _extract_resources(tfplan, filename) + return _generate_tf_content(planned_resources) + + +def _generate_tf_content(resource_changes: list[ResourceChange]) -> str: + tf_content = '' + for resource_change in resource_changes: + if not any(item in resource_change.actions for item in ACTIONS_TO_OMIT_RESOURCE): + tf_content += _generate_resource_content(resource_change) + return tf_content + + +def _generate_resource_content(resource_change: ResourceChange) -> str: + resource_content = f'resource "{resource_change.resource_type}" "{_get_resource_name(resource_change)}" {{\n' + if resource_change.values is not None: + for key, value in resource_change.values.items(): + resource_content += f' {key} = {json.dumps(value)}\n' + resource_content += '}\n\n' + return resource_content + + +def _get_resource_name(resource_change: ResourceChange) -> str: + parts = [resource_change.module_address, resource_change.name] + + if resource_change.index is not None: + parts.append(str(resource_change.index)) + + valid_parts = [part for part in parts if part] + + return '.'.join(valid_parts) + + +def _extract_resources(tfplan: str, filename: str) -> list[ResourceChange]: + tfplan_json = load_json(tfplan) + resources: list[ResourceChange] = [] + try: + resource_changes = tfplan_json['resource_changes'] + for resource_change in resource_changes: + resources.append( + ResourceChange( + module_address=resource_change.get('module_address'), + resource_type=resource_change['type'], + name=resource_change['name'], + index=resource_change.get('index'), + actions=resource_change['change']['actions'], + values=resource_change['change']['after'], + ) + ) + except (KeyError, TypeError) as e: + raise TfplanKeyError(filename) from e + return resources diff --git a/cycode/cli/files_collector/models/__init__.py b/cycode/cli/files_collector/models/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cycode/cli/files_collector/models/in_memory_zip.py b/cycode/cli/files_collector/models/in_memory_zip.py new file mode 100644 index 00000000..8bb9bf9e --- /dev/null +++ b/cycode/cli/files_collector/models/in_memory_zip.py @@ -0,0 +1,56 @@ +from collections import defaultdict +from io import BytesIO +from pathlib import Path +from sys import getsizeof +from typing import Optional +from zipfile import ZIP_DEFLATED, ZipFile + +from cycode.cli.user_settings.configuration_manager import ConfigurationManager +from cycode.cli.utils.path_utils import concat_unique_id + + +class InMemoryZip: + def __init__(self) -> None: + self.configuration_manager = ConfigurationManager() + + self.in_memory_zip = BytesIO() + self.zip = ZipFile(self.in_memory_zip, mode='a', compression=ZIP_DEFLATED, allowZip64=False) + + self._files_count = 0 + self._extension_statistics = defaultdict(int) + + def append(self, filename: str, unique_id: Optional[str], content: str) -> None: + self._files_count += 1 + self._extension_statistics[Path(filename).suffix] += 1 + + if unique_id: + filename = concat_unique_id(filename, unique_id) + + # Encode content to bytes with error handling to handle surrogate characters + # that cannot be encoded to UTF-8. Use 'replace' to replace invalid characters + # with the Unicode replacement character (U+FFFD). + content_bytes = content.encode('utf-8', errors='replace') + self.zip.writestr(filename, content_bytes) + + def close(self) -> None: + self.zip.close() + + def read(self) -> bytes: + self.in_memory_zip.seek(0) + return self.in_memory_zip.read() + + def write_on_disk(self, path: 'Path') -> None: + with open(path, 'wb') as f: + f.write(self.read()) + + @property + def size(self) -> int: + return getsizeof(self.in_memory_zip) + + @property + def files_count(self) -> int: + return self._files_count + + @property + def extension_statistics(self) -> dict[str, int]: + return dict(self._extension_statistics) diff --git a/cycode/cli/files_collector/path_documents.py b/cycode/cli/files_collector/path_documents.py new file mode 100644 index 00000000..142c63bf --- /dev/null +++ b/cycode/cli/files_collector/path_documents.py @@ -0,0 +1,127 @@ +import os +from collections.abc import Generator +from typing import TYPE_CHECKING + +from cycode.cli.files_collector.file_excluder import excluder +from cycode.cli.files_collector.iac.tf_content_generator import ( + generate_tf_content_from_tfplan, + generate_tfplan_document_name, + is_iac, + is_tfplan_file, +) +from cycode.cli.files_collector.walk_ignore import walk_ignore +from cycode.cli.logger import logger +from cycode.cli.models import Document +from cycode.cli.utils.path_utils import get_absolute_path, get_file_content + +if TYPE_CHECKING: + from cycode.cli.utils.progress_bar import BaseProgressBar, ProgressBarSection + + +def _get_all_existing_files_in_directory( + path: str, *, walk_with_ignore_patterns: bool = True, is_cycodeignore_allowed: bool = True +) -> list[str]: + files: list[str] = [] + + if walk_with_ignore_patterns: + + def walk_func(path: str) -> Generator[tuple[str, list[str], list[str]], None, None]: + return walk_ignore(path, is_cycodeignore_allowed=is_cycodeignore_allowed) + else: + walk_func = os.walk + + for root, _, filenames in walk_func(path): + for filename in filenames: + files.append(os.path.join(root, filename)) + + return files + + +def _get_relevant_files_in_path(path: str, *, is_cycodeignore_allowed: bool = True) -> list[str]: + absolute_path = get_absolute_path(path) + + if not os.path.isfile(absolute_path) and not os.path.isdir(absolute_path): + raise FileNotFoundError(f'the specified path was not found, path: {absolute_path}') + + if os.path.isfile(absolute_path): + return [absolute_path] + + file_paths = _get_all_existing_files_in_directory(absolute_path, is_cycodeignore_allowed=is_cycodeignore_allowed) + return [file_path for file_path in file_paths if os.path.isfile(file_path)] + + +def _get_relevant_files( + progress_bar: 'BaseProgressBar', + progress_bar_section: 'ProgressBarSection', + scan_type: str, + paths: tuple[str, ...], + *, + is_cycodeignore_allowed: bool = True, +) -> list[str]: + all_files_to_scan = [] + for path in paths: + all_files_to_scan.extend(_get_relevant_files_in_path(path, is_cycodeignore_allowed=is_cycodeignore_allowed)) + + # we are double the progress bar section length because we are going to process the files twice + # first time to get the file list with respect of excluded patterns (excluding takes seconds to execute) + # second time to get the files content + progress_bar_section_len = len(all_files_to_scan) * 2 + progress_bar.set_section_length(progress_bar_section, progress_bar_section_len) + + relevant_files_to_scan = excluder.exclude_irrelevant_files( + progress_bar, progress_bar_section, scan_type, all_files_to_scan + ) + + # after finishing the first processing (excluding), + # we must update the progress bar stage with respect of excluded files. + # now it's possible that we will not process x2 of the files count + # because some of them were excluded, we should subtract the excluded files count + # from the progress bar section length + excluded_files_count = len(all_files_to_scan) - len(relevant_files_to_scan) + progress_bar_section_len = progress_bar_section_len - excluded_files_count + progress_bar.set_section_length(progress_bar_section, progress_bar_section_len) + + logger.debug( + 'Found all relevant files for scanning, %s', {'paths': paths, 'file_to_scan_count': len(relevant_files_to_scan)} + ) + + return relevant_files_to_scan + + +def _generate_document(file: str, scan_type: str, content: str, is_git_diff: bool) -> Document: + if is_iac(scan_type) and is_tfplan_file(file, content): + return _handle_tfplan_file(file, content, is_git_diff) + + return Document(file, content, is_git_diff, absolute_path=file) + + +def _handle_tfplan_file(file: str, content: str, is_git_diff: bool) -> Document: + document_name = generate_tfplan_document_name(file) + tf_content = generate_tf_content_from_tfplan(file, content) + return Document(document_name, tf_content, is_git_diff) + + +def get_relevant_documents( + progress_bar: 'BaseProgressBar', + progress_bar_section: 'ProgressBarSection', + scan_type: str, + paths: tuple[str, ...], + *, + is_git_diff: bool = False, + is_cycodeignore_allowed: bool = True, +) -> list[Document]: + relevant_files = _get_relevant_files( + progress_bar, progress_bar_section, scan_type, paths, is_cycodeignore_allowed=is_cycodeignore_allowed + ) + + documents: list[Document] = [] + for file in relevant_files: + progress_bar.update(progress_bar_section) + + content = get_file_content(file) + if not content: + continue + + documents.append(_generate_document(file, scan_type, content, is_git_diff)) + + return documents diff --git a/cycode/cli/files_collector/repository_documents.py b/cycode/cli/files_collector/repository_documents.py new file mode 100644 index 00000000..935d3db1 --- /dev/null +++ b/cycode/cli/files_collector/repository_documents.py @@ -0,0 +1,26 @@ +from collections.abc import Iterator +from typing import TYPE_CHECKING, Optional, Union + +from cycode.cli.utils.git_proxy import git_proxy + +if TYPE_CHECKING: + from git import Blob, Repo + from git.objects.base import IndexObjUnion + from git.objects.tree import TraversedTreeTup + + +def _should_process_git_object(obj: 'Blob', _: int) -> bool: + return obj.type == 'blob' and obj.size > 0 + + +def get_git_repository_tree_file_entries( + path: str, branch: str +) -> Union[Iterator['IndexObjUnion'], Iterator['TraversedTreeTup']]: + return git_proxy.get_repo(path).tree(branch).traverse(predicate=_should_process_git_object) + + +def get_file_content_from_commit_path(repo: 'Repo', commit: str, file_path: str) -> Optional[str]: + try: + return repo.git.show(f'{commit}:{file_path}') + except git_proxy.get_git_command_error(): + return None diff --git a/cycode/cli/files_collector/sca/__init__.py b/cycode/cli/files_collector/sca/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cycode/cli/files_collector/sca/base_restore_dependencies.py b/cycode/cli/files_collector/sca/base_restore_dependencies.py new file mode 100644 index 00000000..ac391727 --- /dev/null +++ b/cycode/cli/files_collector/sca/base_restore_dependencies.py @@ -0,0 +1,164 @@ +from abc import ABC, abstractmethod +from pathlib import Path +from typing import Optional + +import typer + +from cycode.cli.models import Document +from cycode.cli.utils.path_utils import get_file_content, get_file_dir, get_path_from_context, join_paths +from cycode.cli.utils.shell_executor import shell +from cycode.logger import get_logger + +logger = get_logger('SCA Restore') + + +def build_dep_tree_path(path: str, generated_file_name: str) -> str: + return join_paths(get_file_dir(path), generated_file_name) + + +def execute_commands( + commands: list[list[str]], + timeout: int, + output_file_path: Optional[str] = None, + working_directory: Optional[str] = None, +) -> Optional[str]: + logger.debug( + 'Executing restore commands, %s', + { + 'commands_count': len(commands), + 'timeout_sec': timeout, + 'working_directory': working_directory, + 'output_file_path': output_file_path, + }, + ) + + if not commands: + return None + + try: + outputs = [] + + for command in commands: + command_output = shell(command=command, timeout=timeout, working_directory=working_directory) + if command_output: + outputs.append(command_output) + + joined_output = '\n'.join(outputs) + + if output_file_path: + with open(output_file_path, 'w', encoding='UTF-8') as output_file: + output_file.writelines(joined_output) + except Exception as e: + logger.debug('Unexpected error during command execution', exc_info=e) + return None + + return joined_output + + +class BaseRestoreDependencies(ABC): + def __init__( + self, ctx: typer.Context, is_git_diff: bool, command_timeout: int, create_output_file_manually: bool = False + ) -> None: + self.ctx = ctx + self.is_git_diff = is_git_diff + self.command_timeout = command_timeout + self.create_output_file_manually = create_output_file_manually + + def restore(self, document: Document) -> Optional[Document]: + return self.try_restore_dependencies(document) + + def get_manifest_file_path(self, document: Document) -> str: + return ( + join_paths(get_path_from_context(self.ctx), document.path) if self.ctx.obj.get('monitor') else document.path + ) + + def try_restore_dependencies(self, document: Document) -> Optional[Document]: + manifest_file_path = self.get_manifest_file_path(document) + restore_file_paths = [ + build_dep_tree_path(document.absolute_path, restore_file_path_item) + for restore_file_path_item in self.get_lock_file_names() + ] + restore_file_path = self.get_any_restore_file_already_exist(document, restore_file_paths) + relative_restore_file_path = build_dep_tree_path( + document.path, self.get_restored_lock_file_name(restore_file_path) + ) + + if not self.verify_restore_file_already_exist(restore_file_path): + output = execute_commands( + commands=self.get_commands(manifest_file_path), + timeout=self.command_timeout, + output_file_path=restore_file_path if self.create_output_file_manually else None, + working_directory=self.get_working_directory(document), + ) + if output is None: # one of the commands failed + return None + else: + logger.debug( + 'Lock file already exists, skipping restore commands, %s', + {'restore_file_path': restore_file_path}, + ) + + restore_file_content = get_file_content(restore_file_path) + logger.debug( + 'Restore file loaded, %s', + { + 'restore_file_path': restore_file_path, + 'content_size': len(restore_file_content) if restore_file_content else 0, + 'content_empty': not restore_file_content, + }, + ) + return Document(relative_restore_file_path, restore_file_content, self.is_git_diff) + + def get_manifest_dir(self, document: Document) -> Optional[str]: + """Return the directory containing the manifest file, resolving monitor-mode paths. + + Uses the same path resolution as get_manifest_file_path() to ensure consistency. + Falls back to document.absolute_path when the resolved manifest path is ambiguous. + """ + manifest_file_path = self.get_manifest_file_path(document) + if manifest_file_path: + parent = Path(manifest_file_path).parent + # Skip '.' (no parent) and filesystem root (its own parent) + if parent != Path('.') and parent != parent.parent: + return str(parent) + + base = document.absolute_path or document.path + if base: + parent = Path(base).parent + if parent != Path('.') and parent != parent.parent: + return str(parent) + + return None + + def get_working_directory(self, document: Document) -> Optional[str]: + return str(Path(document.absolute_path).parent) + + def get_restored_lock_file_name(self, restore_file_path: str) -> str: + return self.get_lock_file_name() + + def get_any_restore_file_already_exist(self, document: Document, restore_file_paths: list[str]) -> str: + for restore_file_path in restore_file_paths: + if Path(restore_file_path).is_file(): + return restore_file_path + + return build_dep_tree_path(document.absolute_path, self.get_lock_file_name()) + + @staticmethod + def verify_restore_file_already_exist(restore_file_path: str) -> bool: + return Path(restore_file_path).is_file() + + @abstractmethod + def is_project(self, document: Document) -> bool: + pass + + @abstractmethod + def get_commands(self, manifest_file_path: str) -> list[list[str]]: + pass + + @abstractmethod + def get_lock_file_name(self) -> str: + pass + + @abstractmethod + def get_lock_file_names(self) -> list[str]: + pass diff --git a/cycode/cli/files_collector/sca/go/__init__.py b/cycode/cli/files_collector/sca/go/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cycode/cli/files_collector/sca/go/restore_go_dependencies.py b/cycode/cli/files_collector/sca/go/restore_go_dependencies.py new file mode 100644 index 00000000..b98fbaf5 --- /dev/null +++ b/cycode/cli/files_collector/sca/go/restore_go_dependencies.py @@ -0,0 +1,50 @@ +from pathlib import Path +from typing import Optional + +import typer + +from cycode.cli.files_collector.sca.base_restore_dependencies import BaseRestoreDependencies +from cycode.cli.models import Document +from cycode.logger import get_logger + +logger = get_logger('Go Restore Dependencies') + +GO_PROJECT_FILE_EXTENSIONS = ['.mod', '.sum'] +GO_RESTORE_FILE_NAME = 'go.mod.graph' +BUILD_GO_FILE_NAME = 'go.mod' +BUILD_GO_LOCK_FILE_NAME = 'go.sum' + + +class RestoreGoDependencies(BaseRestoreDependencies): + def __init__(self, ctx: typer.Context, is_git_diff: bool, command_timeout: int) -> None: + super().__init__(ctx, is_git_diff, command_timeout, create_output_file_manually=True) + + def try_restore_dependencies(self, document: Document) -> Optional[Document]: + manifest_exists = (Path(self.get_working_directory(document)) / BUILD_GO_FILE_NAME).is_file() + lock_exists = (Path(self.get_working_directory(document)) / BUILD_GO_LOCK_FILE_NAME).is_file() + + if not manifest_exists or not lock_exists: + logger.info('No manifest go.mod file found' if not manifest_exists else 'No manifest go.sum file found') + + manifest_files_exists = manifest_exists and lock_exists + + if not manifest_files_exists: + return None + + return super().try_restore_dependencies(document) + + def is_project(self, document: Document) -> bool: + return any(document.path.endswith(ext) for ext in GO_PROJECT_FILE_EXTENSIONS) + + def get_commands(self, manifest_file_path: str) -> list[list[str]]: + return [ + ['go', 'list', '-m', '-json', 'all'], + ['echo', '------------------------------------------------------'], + ['go', 'mod', 'graph'], + ] + + def get_lock_file_name(self) -> str: + return GO_RESTORE_FILE_NAME + + def get_lock_file_names(self) -> list[str]: + return [self.get_lock_file_name()] diff --git a/cycode/cli/files_collector/sca/maven/__init__.py b/cycode/cli/files_collector/sca/maven/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cycode/cli/files_collector/sca/maven/restore_gradle_dependencies.py b/cycode/cli/files_collector/sca/maven/restore_gradle_dependencies.py new file mode 100644 index 00000000..d2687bf6 --- /dev/null +++ b/cycode/cli/files_collector/sca/maven/restore_gradle_dependencies.py @@ -0,0 +1,68 @@ +import os +import re +from typing import Optional + +import typer + +from cycode.cli.files_collector.sca.base_restore_dependencies import BaseRestoreDependencies +from cycode.cli.models import Document +from cycode.cli.utils.path_utils import get_path_from_context +from cycode.cli.utils.shell_executor import shell + +BUILD_GRADLE_FILE_NAME = 'build.gradle' +BUILD_GRADLE_KTS_FILE_NAME = 'build.gradle.kts' +BUILD_GRADLE_DEP_TREE_FILE_NAME = 'gradle-dependencies-generated.txt' +BUILD_GRADLE_ALL_PROJECTS_COMMAND = ['gradle', 'projects'] +ALL_PROJECTS_REGEX = r"[+-]{3} Project '(.*?)'" + + +class RestoreGradleDependencies(BaseRestoreDependencies): + def __init__( + self, ctx: typer.Context, is_git_diff: bool, command_timeout: int, projects: Optional[set[str]] = None + ) -> None: + super().__init__(ctx, is_git_diff, command_timeout, create_output_file_manually=True) + if projects is None: + projects = set() + self.projects = self.get_all_projects() if self.is_gradle_sub_projects() else projects + + def is_gradle_sub_projects(self) -> bool: + return self.ctx.obj.get('gradle_all_sub_projects', False) + + def is_project(self, document: Document) -> bool: + return document.path.endswith(BUILD_GRADLE_FILE_NAME) or document.path.endswith(BUILD_GRADLE_KTS_FILE_NAME) + + def get_commands(self, manifest_file_path: str) -> list[list[str]]: + return ( + self.get_commands_for_sub_projects(manifest_file_path) + if self.is_gradle_sub_projects() + else [['gradle', 'dependencies', '-b', manifest_file_path, '-q', '--console', 'plain']] + ) + + def get_lock_file_name(self) -> str: + return BUILD_GRADLE_DEP_TREE_FILE_NAME + + def get_lock_file_names(self) -> list[str]: + return [self.get_lock_file_name()] + + def get_working_directory(self, document: Document) -> Optional[str]: + return get_path_from_context(self.ctx) if self.is_gradle_sub_projects() else None + + def get_all_projects(self) -> set[str]: + output = shell( + command=BUILD_GRADLE_ALL_PROJECTS_COMMAND, + timeout=self.command_timeout, + working_directory=get_path_from_context(self.ctx), + ) + if not output: + return set() + + return set(re.findall(ALL_PROJECTS_REGEX, output)) + + def get_commands_for_sub_projects(self, manifest_file_path: str) -> list[list[str]]: + project_name = os.path.basename(os.path.dirname(manifest_file_path)) + project_name = f':{project_name}' + return ( + [['gradle', f'{project_name}:dependencies', '-q', '--console', 'plain']] + if project_name in self.projects + else [] + ) diff --git a/cycode/cli/files_collector/sca/maven/restore_maven_dependencies.py b/cycode/cli/files_collector/sca/maven/restore_maven_dependencies.py new file mode 100644 index 00000000..34499bdf --- /dev/null +++ b/cycode/cli/files_collector/sca/maven/restore_maven_dependencies.py @@ -0,0 +1,87 @@ +from os import path +from typing import Optional + +import typer + +from cycode.cli.files_collector.sca.base_restore_dependencies import ( + BaseRestoreDependencies, + build_dep_tree_path, + execute_commands, +) +from cycode.cli.models import Document +from cycode.cli.utils.path_utils import get_file_content, get_file_dir, join_paths + +BUILD_MAVEN_FILE_NAME = 'pom.xml' +MAVEN_CYCLONE_DEP_TREE_FILE_NAME = 'bom.json' +MAVEN_DEP_TREE_FILE_NAME = 'bcde.mvndeps' + + +class RestoreMavenDependencies(BaseRestoreDependencies): + def __init__(self, ctx: typer.Context, is_git_diff: bool, command_timeout: int) -> None: + super().__init__(ctx, is_git_diff, command_timeout) + + def is_project(self, document: Document) -> bool: + return path.basename(document.path).split('/')[-1] == BUILD_MAVEN_FILE_NAME + + def get_commands(self, manifest_file_path: str) -> list[list[str]]: + command = ['mvn', 'org.cyclonedx:cyclonedx-maven-plugin:2.9.1:makeAggregateBom', '-f', manifest_file_path] + + maven_settings_file = self.ctx.obj.get('maven_settings_file') + if maven_settings_file: + command += ['-s', str(maven_settings_file)] + return [command] + + def get_lock_file_name(self) -> str: + return join_paths('target', MAVEN_CYCLONE_DEP_TREE_FILE_NAME) + + def get_lock_file_names(self) -> list[str]: + return [self.get_lock_file_name()] + + def try_restore_dependencies(self, document: Document) -> Optional[Document]: + manifest_file_path = self.get_manifest_file_path(document) + if document.content is None: + return self.restore_from_secondary_command(document, manifest_file_path) + + restore_dependencies_document = super().try_restore_dependencies(document) + if restore_dependencies_document is None: + return None + + restore_dependencies_document.content = get_file_content( + join_paths(get_file_dir(manifest_file_path), self.get_lock_file_name()) + ) + + return restore_dependencies_document + + def restore_from_secondary_command(self, document: Document, manifest_file_path: str) -> Optional[Document]: + restore_content = execute_commands( + commands=self.create_secondary_restore_commands(manifest_file_path), + timeout=self.command_timeout, + working_directory=self.get_working_directory(document), + ) + if restore_content is None: + return None + + restore_file_path = build_dep_tree_path(document.absolute_path, MAVEN_DEP_TREE_FILE_NAME) + return Document( + path=build_dep_tree_path(document.path, MAVEN_DEP_TREE_FILE_NAME), + content=get_file_content(restore_file_path), + is_git_diff_format=self.is_git_diff, + absolute_path=restore_file_path, + ) + + def create_secondary_restore_commands(self, manifest_file_path: str) -> list[list[str]]: + command = [ + 'mvn', + 'dependency:tree', + '-B', + '-DoutputType=text', + '-f', + manifest_file_path, + f'-DoutputFile={MAVEN_DEP_TREE_FILE_NAME}', + ] + + maven_settings_file = self.ctx.obj.get('maven_settings_file') + if maven_settings_file: + command += ['-s', str(maven_settings_file)] + + return [command] diff --git a/cycode/cli/files_collector/sca/npm/__init__.py b/cycode/cli/files_collector/sca/npm/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cycode/cli/files_collector/sca/npm/restore_deno_dependencies.py b/cycode/cli/files_collector/sca/npm/restore_deno_dependencies.py new file mode 100644 index 00000000..d3aeb5e5 --- /dev/null +++ b/cycode/cli/files_collector/sca/npm/restore_deno_dependencies.py @@ -0,0 +1,46 @@ +from pathlib import Path +from typing import Optional + +import typer + +from cycode.cli.files_collector.sca.base_restore_dependencies import BaseRestoreDependencies, build_dep_tree_path +from cycode.cli.models import Document +from cycode.cli.utils.path_utils import get_file_content +from cycode.logger import get_logger + +logger = get_logger('Deno Restore Dependencies') + +DENO_MANIFEST_FILE_NAMES = ('deno.json', 'deno.jsonc') +DENO_LOCK_FILE_NAME = 'deno.lock' + + +class RestoreDenoDependencies(BaseRestoreDependencies): + def __init__(self, ctx: typer.Context, is_git_diff: bool, command_timeout: int) -> None: + super().__init__(ctx, is_git_diff, command_timeout) + + def is_project(self, document: Document) -> bool: + return Path(document.path).name in DENO_MANIFEST_FILE_NAMES + + def try_restore_dependencies(self, document: Document) -> Optional[Document]: + manifest_dir = self.get_manifest_dir(document) + if not manifest_dir: + return None + + lockfile_path = Path(manifest_dir) / DENO_LOCK_FILE_NAME + if not lockfile_path.is_file(): + logger.debug('No deno.lock found alongside deno.json, skipping deno restore, %s', {'path': document.path}) + return None + + content = get_file_content(str(lockfile_path)) + relative_path = build_dep_tree_path(document.path, DENO_LOCK_FILE_NAME) + logger.debug('Using existing deno.lock, %s', {'path': str(lockfile_path)}) + return Document(relative_path, content, self.is_git_diff) + + def get_commands(self, manifest_file_path: str) -> list[list[str]]: + return [] + + def get_lock_file_name(self) -> str: + return DENO_LOCK_FILE_NAME + + def get_lock_file_names(self) -> list[str]: + return [DENO_LOCK_FILE_NAME] diff --git a/cycode/cli/files_collector/sca/npm/restore_npm_dependencies.py b/cycode/cli/files_collector/sca/npm/restore_npm_dependencies.py new file mode 100644 index 00000000..d07bc4a5 --- /dev/null +++ b/cycode/cli/files_collector/sca/npm/restore_npm_dependencies.py @@ -0,0 +1,67 @@ +from pathlib import Path + +import typer + +from cycode.cli.files_collector.sca.base_restore_dependencies import BaseRestoreDependencies +from cycode.cli.models import Document +from cycode.logger import get_logger + +logger = get_logger('NPM Restore Dependencies') + +NPM_MANIFEST_FILE_NAME = 'package.json' +NPM_LOCK_FILE_NAME = 'package-lock.json' +# These lockfiles indicate another package manager owns the project — NPM should not run +_ALTERNATIVE_LOCK_FILES = ('yarn.lock', 'pnpm-lock.yaml', 'deno.lock') + + +class RestoreNpmDependencies(BaseRestoreDependencies): + def __init__(self, ctx: typer.Context, is_git_diff: bool, command_timeout: int) -> None: + super().__init__(ctx, is_git_diff, command_timeout) + + def is_project(self, document: Document) -> bool: + """Match only package.json files that are not managed by Yarn or pnpm. + + Yarn and pnpm projects are handled by their dedicated handlers, which run before + this one in the handler list. This handler is the npm fallback. + """ + if Path(document.path).name != NPM_MANIFEST_FILE_NAME: + return False + + manifest_dir = self.get_manifest_dir(document) + if manifest_dir: + for lock_file in _ALTERNATIVE_LOCK_FILES: + if (Path(manifest_dir) / lock_file).is_file(): + logger.debug( + 'Skipping npm restore: alternative lockfile detected, %s', + {'path': document.path, 'lockfile': lock_file}, + ) + return False + + return True + + def get_commands(self, manifest_file_path: str) -> list[list[str]]: + return [ + [ + 'npm', + 'install', + '--prefix', + self.prepare_manifest_file_path_for_command(manifest_file_path), + '--package-lock-only', + '--ignore-scripts', + '--no-audit', + ] + ] + + def get_lock_file_name(self) -> str: + return NPM_LOCK_FILE_NAME + + def get_lock_file_names(self) -> list[str]: + return [NPM_LOCK_FILE_NAME] + + @staticmethod + def prepare_manifest_file_path_for_command(manifest_file_path: str) -> str: + if manifest_file_path.endswith(NPM_MANIFEST_FILE_NAME): + parent = Path(manifest_file_path).parent + dir_path = str(parent) + return dir_path if dir_path and dir_path != '.' else '' + return manifest_file_path diff --git a/cycode/cli/files_collector/sca/npm/restore_pnpm_dependencies.py b/cycode/cli/files_collector/sca/npm/restore_pnpm_dependencies.py new file mode 100644 index 00000000..bce7eff6 --- /dev/null +++ b/cycode/cli/files_collector/sca/npm/restore_pnpm_dependencies.py @@ -0,0 +1,70 @@ +import json +from pathlib import Path +from typing import Optional + +import typer + +from cycode.cli.files_collector.sca.base_restore_dependencies import BaseRestoreDependencies, build_dep_tree_path +from cycode.cli.models import Document +from cycode.cli.utils.path_utils import get_file_content +from cycode.logger import get_logger + +logger = get_logger('Pnpm Restore Dependencies') + +PNPM_MANIFEST_FILE_NAME = 'package.json' +PNPM_LOCK_FILE_NAME = 'pnpm-lock.yaml' + + +def _indicates_pnpm(package_json_content: Optional[str]) -> bool: + """Return True if package.json content signals that this project uses pnpm.""" + if not package_json_content: + return False + try: + data = json.loads(package_json_content) + except (json.JSONDecodeError, ValueError): + return False + + package_manager = data.get('packageManager', '') + if isinstance(package_manager, str) and package_manager.startswith('pnpm'): + return True + + engines = data.get('engines', {}) + return isinstance(engines, dict) and 'pnpm' in engines + + +class RestorePnpmDependencies(BaseRestoreDependencies): + def __init__(self, ctx: typer.Context, is_git_diff: bool, command_timeout: int) -> None: + super().__init__(ctx, is_git_diff, command_timeout) + + def is_project(self, document: Document) -> bool: + if Path(document.path).name != PNPM_MANIFEST_FILE_NAME: + return False + + manifest_dir = self.get_manifest_dir(document) + if manifest_dir and (Path(manifest_dir) / PNPM_LOCK_FILE_NAME).is_file(): + return True + + return _indicates_pnpm(document.content) + + def try_restore_dependencies(self, document: Document) -> Optional[Document]: + manifest_dir = self.get_manifest_dir(document) + lockfile_path = Path(manifest_dir) / PNPM_LOCK_FILE_NAME if manifest_dir else None + + if lockfile_path and lockfile_path.is_file(): + # Lockfile already exists — read it directly without running pnpm + content = get_file_content(str(lockfile_path)) + relative_path = build_dep_tree_path(document.path, PNPM_LOCK_FILE_NAME) + logger.debug('Using existing pnpm-lock.yaml, %s', {'path': str(lockfile_path)}) + return Document(relative_path, content, self.is_git_diff) + + # Lockfile absent but pnpm is indicated in package.json — generate it + return super().try_restore_dependencies(document) + + def get_commands(self, manifest_file_path: str) -> list[list[str]]: + return [['pnpm', 'install', '--ignore-scripts']] + + def get_lock_file_name(self) -> str: + return PNPM_LOCK_FILE_NAME + + def get_lock_file_names(self) -> list[str]: + return [PNPM_LOCK_FILE_NAME] diff --git a/cycode/cli/files_collector/sca/npm/restore_yarn_dependencies.py b/cycode/cli/files_collector/sca/npm/restore_yarn_dependencies.py new file mode 100644 index 00000000..79b0c4ec --- /dev/null +++ b/cycode/cli/files_collector/sca/npm/restore_yarn_dependencies.py @@ -0,0 +1,70 @@ +import json +from pathlib import Path +from typing import Optional + +import typer + +from cycode.cli.files_collector.sca.base_restore_dependencies import BaseRestoreDependencies, build_dep_tree_path +from cycode.cli.models import Document +from cycode.cli.utils.path_utils import get_file_content +from cycode.logger import get_logger + +logger = get_logger('Yarn Restore Dependencies') + +YARN_MANIFEST_FILE_NAME = 'package.json' +YARN_LOCK_FILE_NAME = 'yarn.lock' + + +def _indicates_yarn(package_json_content: Optional[str]) -> bool: + """Return True if package.json content signals that this project uses Yarn.""" + if not package_json_content: + return False + try: + data = json.loads(package_json_content) + except (json.JSONDecodeError, ValueError): + return False + + package_manager = data.get('packageManager', '') + if isinstance(package_manager, str) and package_manager.startswith('yarn'): + return True + + engines = data.get('engines', {}) + return isinstance(engines, dict) and 'yarn' in engines + + +class RestoreYarnDependencies(BaseRestoreDependencies): + def __init__(self, ctx: typer.Context, is_git_diff: bool, command_timeout: int) -> None: + super().__init__(ctx, is_git_diff, command_timeout) + + def is_project(self, document: Document) -> bool: + if Path(document.path).name != YARN_MANIFEST_FILE_NAME: + return False + + manifest_dir = self.get_manifest_dir(document) + if manifest_dir and (Path(manifest_dir) / YARN_LOCK_FILE_NAME).is_file(): + return True + + return _indicates_yarn(document.content) + + def try_restore_dependencies(self, document: Document) -> Optional[Document]: + manifest_dir = self.get_manifest_dir(document) + lockfile_path = Path(manifest_dir) / YARN_LOCK_FILE_NAME if manifest_dir else None + + if lockfile_path and lockfile_path.is_file(): + # Lockfile already exists — read it directly without running yarn + content = get_file_content(str(lockfile_path)) + relative_path = build_dep_tree_path(document.path, YARN_LOCK_FILE_NAME) + logger.debug('Using existing yarn.lock, %s', {'path': str(lockfile_path)}) + return Document(relative_path, content, self.is_git_diff) + + # Lockfile absent but yarn is indicated in package.json — generate it + return super().try_restore_dependencies(document) + + def get_commands(self, manifest_file_path: str) -> list[list[str]]: + return [['yarn', 'install', '--ignore-scripts']] + + def get_lock_file_name(self) -> str: + return YARN_LOCK_FILE_NAME + + def get_lock_file_names(self) -> list[str]: + return [YARN_LOCK_FILE_NAME] diff --git a/cycode/cli/files_collector/sca/nuget/__init__.py b/cycode/cli/files_collector/sca/nuget/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cycode/cli/files_collector/sca/nuget/restore_nuget_dependencies.py b/cycode/cli/files_collector/sca/nuget/restore_nuget_dependencies.py new file mode 100644 index 00000000..95ced0ff --- /dev/null +++ b/cycode/cli/files_collector/sca/nuget/restore_nuget_dependencies.py @@ -0,0 +1,24 @@ +import typer + +from cycode.cli.files_collector.sca.base_restore_dependencies import BaseRestoreDependencies +from cycode.cli.models import Document + +NUGET_PROJECT_FILE_EXTENSIONS = ['.csproj', '.vbproj'] +NUGET_LOCK_FILE_NAME = 'packages.lock.json' + + +class RestoreNugetDependencies(BaseRestoreDependencies): + def __init__(self, ctx: typer.Context, is_git_diff: bool, command_timeout: int) -> None: + super().__init__(ctx, is_git_diff, command_timeout) + + def is_project(self, document: Document) -> bool: + return any(document.path.endswith(ext) for ext in NUGET_PROJECT_FILE_EXTENSIONS) + + def get_commands(self, manifest_file_path: str) -> list[list[str]]: + return [['dotnet', 'restore', manifest_file_path, '--use-lock-file', '--verbosity', 'quiet']] + + def get_lock_file_name(self) -> str: + return NUGET_LOCK_FILE_NAME + + def get_lock_file_names(self) -> list[str]: + return [self.get_lock_file_name()] diff --git a/cycode/cli/files_collector/sca/php/__init__.py b/cycode/cli/files_collector/sca/php/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cycode/cli/files_collector/sca/php/restore_composer_dependencies.py b/cycode/cli/files_collector/sca/php/restore_composer_dependencies.py new file mode 100644 index 00000000..98b3564c --- /dev/null +++ b/cycode/cli/files_collector/sca/php/restore_composer_dependencies.py @@ -0,0 +1,54 @@ +from pathlib import Path +from typing import Optional + +import typer + +from cycode.cli.files_collector.sca.base_restore_dependencies import BaseRestoreDependencies, build_dep_tree_path +from cycode.cli.models import Document +from cycode.cli.utils.path_utils import get_file_content +from cycode.logger import get_logger + +logger = get_logger('Composer Restore Dependencies') + +COMPOSER_MANIFEST_FILE_NAME = 'composer.json' +COMPOSER_LOCK_FILE_NAME = 'composer.lock' + + +class RestoreComposerDependencies(BaseRestoreDependencies): + def __init__(self, ctx: typer.Context, is_git_diff: bool, command_timeout: int) -> None: + super().__init__(ctx, is_git_diff, command_timeout) + + def is_project(self, document: Document) -> bool: + return Path(document.path).name == COMPOSER_MANIFEST_FILE_NAME + + def try_restore_dependencies(self, document: Document) -> Optional[Document]: + manifest_dir = self.get_manifest_dir(document) + lockfile_path = Path(manifest_dir) / COMPOSER_LOCK_FILE_NAME if manifest_dir else None + + if lockfile_path and lockfile_path.is_file(): + # Lockfile already exists — read it directly without running composer + content = get_file_content(str(lockfile_path)) + relative_path = build_dep_tree_path(document.path, COMPOSER_LOCK_FILE_NAME) + logger.debug('Using existing composer.lock, %s', {'path': str(lockfile_path)}) + return Document(relative_path, content, self.is_git_diff) + + # Lockfile absent — generate it + return super().try_restore_dependencies(document) + + def get_commands(self, manifest_file_path: str) -> list[list[str]]: + return [ + [ + 'composer', + 'update', + '--no-cache', + '--no-install', + '--no-scripts', + '--ignore-platform-reqs', + ] + ] + + def get_lock_file_name(self) -> str: + return COMPOSER_LOCK_FILE_NAME + + def get_lock_file_names(self) -> list[str]: + return [COMPOSER_LOCK_FILE_NAME] diff --git a/cycode/cli/files_collector/sca/python/__init__.py b/cycode/cli/files_collector/sca/python/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cycode/cli/files_collector/sca/python/restore_pipenv_dependencies.py b/cycode/cli/files_collector/sca/python/restore_pipenv_dependencies.py new file mode 100644 index 00000000..df91707c --- /dev/null +++ b/cycode/cli/files_collector/sca/python/restore_pipenv_dependencies.py @@ -0,0 +1,45 @@ +from pathlib import Path +from typing import Optional + +import typer + +from cycode.cli.files_collector.sca.base_restore_dependencies import BaseRestoreDependencies, build_dep_tree_path +from cycode.cli.models import Document +from cycode.cli.utils.path_utils import get_file_content +from cycode.logger import get_logger + +logger = get_logger('Pipenv Restore Dependencies') + +PIPENV_MANIFEST_FILE_NAME = 'Pipfile' +PIPENV_LOCK_FILE_NAME = 'Pipfile.lock' + + +class RestorePipenvDependencies(BaseRestoreDependencies): + def __init__(self, ctx: typer.Context, is_git_diff: bool, command_timeout: int) -> None: + super().__init__(ctx, is_git_diff, command_timeout) + + def is_project(self, document: Document) -> bool: + return Path(document.path).name == PIPENV_MANIFEST_FILE_NAME + + def try_restore_dependencies(self, document: Document) -> Optional[Document]: + manifest_dir = self.get_manifest_dir(document) + lockfile_path = Path(manifest_dir) / PIPENV_LOCK_FILE_NAME if manifest_dir else None + + if lockfile_path and lockfile_path.is_file(): + # Lockfile already exists — read it directly without running pipenv + content = get_file_content(str(lockfile_path)) + relative_path = build_dep_tree_path(document.path, PIPENV_LOCK_FILE_NAME) + logger.debug('Using existing Pipfile.lock, %s', {'path': str(lockfile_path)}) + return Document(relative_path, content, self.is_git_diff) + + # Lockfile absent — generate it + return super().try_restore_dependencies(document) + + def get_commands(self, manifest_file_path: str) -> list[list[str]]: + return [['pipenv', 'lock']] + + def get_lock_file_name(self) -> str: + return PIPENV_LOCK_FILE_NAME + + def get_lock_file_names(self) -> list[str]: + return [PIPENV_LOCK_FILE_NAME] diff --git a/cycode/cli/files_collector/sca/python/restore_poetry_dependencies.py b/cycode/cli/files_collector/sca/python/restore_poetry_dependencies.py new file mode 100644 index 00000000..f681bd63 --- /dev/null +++ b/cycode/cli/files_collector/sca/python/restore_poetry_dependencies.py @@ -0,0 +1,62 @@ +from pathlib import Path +from typing import Optional + +import typer + +from cycode.cli.files_collector.sca.base_restore_dependencies import BaseRestoreDependencies, build_dep_tree_path +from cycode.cli.models import Document +from cycode.cli.utils.path_utils import get_file_content +from cycode.logger import get_logger + +logger = get_logger('Poetry Restore Dependencies') + +POETRY_MANIFEST_FILE_NAME = 'pyproject.toml' +POETRY_LOCK_FILE_NAME = 'poetry.lock' + +# Section header that signals this pyproject.toml is managed by Poetry +_POETRY_TOOL_SECTION = '[tool.poetry]' + + +def _indicates_poetry(pyproject_content: Optional[str]) -> bool: + """Return True if pyproject.toml content signals that this project uses Poetry.""" + if not pyproject_content: + return False + return _POETRY_TOOL_SECTION in pyproject_content + + +class RestorePoetryDependencies(BaseRestoreDependencies): + def __init__(self, ctx: typer.Context, is_git_diff: bool, command_timeout: int) -> None: + super().__init__(ctx, is_git_diff, command_timeout) + + def is_project(self, document: Document) -> bool: + if Path(document.path).name != POETRY_MANIFEST_FILE_NAME: + return False + + manifest_dir = self.get_manifest_dir(document) + if manifest_dir and (Path(manifest_dir) / POETRY_LOCK_FILE_NAME).is_file(): + return True + + return _indicates_poetry(document.content) + + def try_restore_dependencies(self, document: Document) -> Optional[Document]: + manifest_dir = self.get_manifest_dir(document) + lockfile_path = Path(manifest_dir) / POETRY_LOCK_FILE_NAME if manifest_dir else None + + if lockfile_path and lockfile_path.is_file(): + # Lockfile already exists — read it directly without running poetry + content = get_file_content(str(lockfile_path)) + relative_path = build_dep_tree_path(document.path, POETRY_LOCK_FILE_NAME) + logger.debug('Using existing poetry.lock, %s', {'path': str(lockfile_path)}) + return Document(relative_path, content, self.is_git_diff) + + # Lockfile absent but Poetry is indicated in pyproject.toml — generate it + return super().try_restore_dependencies(document) + + def get_commands(self, manifest_file_path: str) -> list[list[str]]: + return [['poetry', 'lock']] + + def get_lock_file_name(self) -> str: + return POETRY_LOCK_FILE_NAME + + def get_lock_file_names(self) -> list[str]: + return [POETRY_LOCK_FILE_NAME] diff --git a/cycode/cli/files_collector/sca/ruby/__init__.py b/cycode/cli/files_collector/sca/ruby/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cycode/cli/files_collector/sca/ruby/restore_ruby_dependencies.py b/cycode/cli/files_collector/sca/ruby/restore_ruby_dependencies.py new file mode 100644 index 00000000..a8358270 --- /dev/null +++ b/cycode/cli/files_collector/sca/ruby/restore_ruby_dependencies.py @@ -0,0 +1,19 @@ +from cycode.cli.files_collector.sca.base_restore_dependencies import BaseRestoreDependencies +from cycode.cli.models import Document + +RUBY_PROJECT_FILE_EXTENSIONS = ['Gemfile'] +RUBY_LOCK_FILE_NAME = 'Gemfile.lock' + + +class RestoreRubyDependencies(BaseRestoreDependencies): + def is_project(self, document: Document) -> bool: + return any(document.path.endswith(ext) for ext in RUBY_PROJECT_FILE_EXTENSIONS) + + def get_commands(self, manifest_file_path: str) -> list[list[str]]: + return [['bundle', '--quiet']] + + def get_lock_file_name(self) -> str: + return RUBY_LOCK_FILE_NAME + + def get_lock_file_names(self) -> list[str]: + return [self.get_lock_file_name()] diff --git a/cycode/cli/files_collector/sca/sbt/__init__.py b/cycode/cli/files_collector/sca/sbt/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cycode/cli/files_collector/sca/sbt/restore_sbt_dependencies.py b/cycode/cli/files_collector/sca/sbt/restore_sbt_dependencies.py new file mode 100644 index 00000000..c9529d6a --- /dev/null +++ b/cycode/cli/files_collector/sca/sbt/restore_sbt_dependencies.py @@ -0,0 +1,19 @@ +from cycode.cli.files_collector.sca.base_restore_dependencies import BaseRestoreDependencies +from cycode.cli.models import Document + +SBT_PROJECT_FILE_EXTENSIONS = ['sbt'] +SBT_LOCK_FILE_NAME = 'build.sbt.lock' + + +class RestoreSbtDependencies(BaseRestoreDependencies): + def is_project(self, document: Document) -> bool: + return any(document.path.endswith(ext) for ext in SBT_PROJECT_FILE_EXTENSIONS) + + def get_commands(self, manifest_file_path: str) -> list[list[str]]: + return [['sbt', 'dependencyLockWrite', '--verbose']] + + def get_lock_file_name(self) -> str: + return SBT_LOCK_FILE_NAME + + def get_lock_file_names(self) -> list[str]: + return [self.get_lock_file_name()] diff --git a/cycode/cli/files_collector/sca/sca_file_collector.py b/cycode/cli/files_collector/sca/sca_file_collector.py new file mode 100644 index 00000000..b194deef --- /dev/null +++ b/cycode/cli/files_collector/sca/sca_file_collector.py @@ -0,0 +1,201 @@ +import os +from typing import TYPE_CHECKING, Optional + +import typer + +from cycode.cli import consts +from cycode.cli.files_collector.repository_documents import get_file_content_from_commit_path +from cycode.cli.files_collector.sca.base_restore_dependencies import BaseRestoreDependencies +from cycode.cli.files_collector.sca.go.restore_go_dependencies import RestoreGoDependencies +from cycode.cli.files_collector.sca.maven.restore_gradle_dependencies import RestoreGradleDependencies +from cycode.cli.files_collector.sca.maven.restore_maven_dependencies import RestoreMavenDependencies +from cycode.cli.files_collector.sca.npm.restore_deno_dependencies import RestoreDenoDependencies +from cycode.cli.files_collector.sca.npm.restore_npm_dependencies import RestoreNpmDependencies +from cycode.cli.files_collector.sca.npm.restore_pnpm_dependencies import RestorePnpmDependencies +from cycode.cli.files_collector.sca.npm.restore_yarn_dependencies import RestoreYarnDependencies +from cycode.cli.files_collector.sca.nuget.restore_nuget_dependencies import RestoreNugetDependencies +from cycode.cli.files_collector.sca.php.restore_composer_dependencies import RestoreComposerDependencies +from cycode.cli.files_collector.sca.python.restore_pipenv_dependencies import RestorePipenvDependencies +from cycode.cli.files_collector.sca.python.restore_poetry_dependencies import RestorePoetryDependencies +from cycode.cli.files_collector.sca.ruby.restore_ruby_dependencies import RestoreRubyDependencies +from cycode.cli.files_collector.sca.sbt.restore_sbt_dependencies import RestoreSbtDependencies +from cycode.cli.models import Document +from cycode.cli.utils.git_proxy import git_proxy +from cycode.cli.utils.path_utils import get_file_content, get_file_dir, get_path_from_context, join_paths +from cycode.logger import get_logger + +if TYPE_CHECKING: + from git import Repo + +BUILD_DEP_TREE_TIMEOUT = 180 + + +logger = get_logger('SCA File Collector') + + +def _add_ecosystem_related_files_if_exists( + documents: list[Document], repo: Optional['Repo'] = None, commit_rev: Optional[str] = None +) -> None: + documents_to_add: list[Document] = [] + for doc in documents: + ecosystem = _get_project_file_ecosystem(doc) + if ecosystem is None: + logger.debug('Failed to resolve project file ecosystem: %s', doc.path) + continue + + documents_to_add.extend(_get_doc_ecosystem_related_project_files(doc, documents, ecosystem, commit_rev, repo)) + + documents.extend(documents_to_add) + + +def perform_sca_pre_commit_range_scan_actions( + path: str, + from_commit_documents: list[Document], + from_commit_rev: str, + to_commit_documents: list[Document], + to_commit_rev: str, +) -> None: + repo = git_proxy.get_repo(path) + _add_ecosystem_related_files_if_exists(from_commit_documents, repo, from_commit_rev) + _add_ecosystem_related_files_if_exists(to_commit_documents, repo, to_commit_rev) + + +def perform_sca_pre_hook_range_scan_actions( + repo_path: str, git_head_documents: list[Document], pre_committed_documents: list[Document] +) -> None: + repo = git_proxy.get_repo(repo_path) + _add_ecosystem_related_files_if_exists(git_head_documents, repo, consts.GIT_HEAD_COMMIT_REV) + _add_ecosystem_related_files_if_exists(pre_committed_documents) + + +def _get_doc_ecosystem_related_project_files( + doc: Document, documents: list[Document], ecosystem: str, commit_rev: Optional[str], repo: Optional['Repo'] +) -> list[Document]: + documents_to_add: list[Document] = [] + for ecosystem_project_file in consts.PROJECT_FILES_BY_ECOSYSTEM_MAP.get(ecosystem): + file_to_search = join_paths(get_file_dir(doc.path), ecosystem_project_file) + if not _is_project_file_exists_in_documents(documents, file_to_search): + if repo: + file_content = get_file_content_from_commit_path(repo, commit_rev, file_to_search) + else: + file_content = get_file_content(file_to_search) + + if file_content is not None: + documents_to_add.append(Document(file_to_search, file_content)) + + return documents_to_add + + +def _is_project_file_exists_in_documents(documents: list[Document], file: str) -> bool: + return any(doc for doc in documents if file == doc.path) + + +def _get_project_file_ecosystem(document: Document) -> Optional[str]: + for ecosystem, project_files in consts.PROJECT_FILES_BY_ECOSYSTEM_MAP.items(): + for project_file in project_files: + if document.path.endswith(project_file): + return ecosystem + return None + + +def _get_manifest_file_path(document: Document, is_monitor_action: bool, project_path: str) -> str: + return join_paths(project_path, document.path) if is_monitor_action else document.path + + +def _try_restore_dependencies( + ctx: typer.Context, + restore_dependencies: 'BaseRestoreDependencies', + document: Document, +) -> Optional[Document]: + if not restore_dependencies.is_project(document): + return None + + restore_dependencies_document = restore_dependencies.restore(document) + if restore_dependencies_document is None: + logger.warning( + 'Error occurred while trying to generate dependencies tree, %s', + {'filename': document.path, 'handler': type(restore_dependencies).__name__}, + ) + return None + + if restore_dependencies_document.content is None: + logger.warning( + 'Error occurred while trying to generate dependencies tree, %s', + {'filename': document.path, 'handler': type(restore_dependencies).__name__}, + ) + restore_dependencies_document.content = '' + else: + is_monitor_action = ctx.obj.get('monitor', False) + project_path = get_path_from_context(ctx) + + manifest_file_path = _get_manifest_file_path(document, is_monitor_action, project_path) + logger.debug('Succeeded to generate dependencies tree on path: %s', manifest_file_path) + + return restore_dependencies_document + + +def _get_restore_handlers(ctx: typer.Context, is_git_diff: bool) -> list[BaseRestoreDependencies]: + build_dep_tree_timeout = int(os.getenv('CYCODE_BUILD_DEP_TREE_TIMEOUT_SECONDS', BUILD_DEP_TREE_TIMEOUT)) + logger.debug( + 'SCA restore handler timeout, %s', + { + 'timeout_sec': build_dep_tree_timeout, + 'source': 'env' if os.getenv('CYCODE_BUILD_DEP_TREE_TIMEOUT_SECONDS') else 'default', + }, + ) + return [ + RestoreGradleDependencies(ctx, is_git_diff, build_dep_tree_timeout), + RestoreMavenDependencies(ctx, is_git_diff, build_dep_tree_timeout), + RestoreSbtDependencies(ctx, is_git_diff, build_dep_tree_timeout), + RestoreGoDependencies(ctx, is_git_diff, build_dep_tree_timeout), + RestoreNugetDependencies(ctx, is_git_diff, build_dep_tree_timeout), + RestoreYarnDependencies(ctx, is_git_diff, build_dep_tree_timeout), + RestorePnpmDependencies(ctx, is_git_diff, build_dep_tree_timeout), + RestoreDenoDependencies(ctx, is_git_diff, build_dep_tree_timeout), + RestoreNpmDependencies(ctx, is_git_diff, build_dep_tree_timeout), # Must be after Yarn & Pnpm for fallback + RestoreRubyDependencies(ctx, is_git_diff, build_dep_tree_timeout), + RestorePoetryDependencies(ctx, is_git_diff, build_dep_tree_timeout), + RestorePipenvDependencies(ctx, is_git_diff, build_dep_tree_timeout), + RestoreComposerDependencies(ctx, is_git_diff, build_dep_tree_timeout), + ] + + +def _add_dependencies_tree_documents( + ctx: typer.Context, documents_to_scan: list[Document], is_git_diff: bool = False +) -> None: + logger.debug( + 'Adding dependencies tree documents, %s', + {'documents_count': len(documents_to_scan), 'is_git_diff': is_git_diff}, + ) + + documents_to_add: dict[str, Document] = {document.path: document for document in documents_to_scan} + restore_dependencies_list = _get_restore_handlers(ctx, is_git_diff) + + for restore_dependencies in restore_dependencies_list: + for document in documents_to_scan: + restore_dependencies_document = _try_restore_dependencies(ctx, restore_dependencies, document) + if restore_dependencies_document is None: + continue + + if restore_dependencies_document.path in documents_to_add: + # Lockfile was already collected during file discovery, so we skip adding it again + logger.debug( + 'Lockfile already exists in scan, skipping duplicate document, %s', + {'path': restore_dependencies_document.path, 'source': 'restore'}, + ) + else: + logger.debug('Adding dependencies tree document, %s', restore_dependencies_document.path) + documents_to_add[restore_dependencies_document.path] = restore_dependencies_document + + logger.debug('Finished adding dependencies tree documents, %s', {'documents_count': len(documents_to_add)}) + + # mutate original list using slice assignment + documents_to_scan[:] = list(documents_to_add.values()) + + +def add_sca_dependencies_tree_documents_if_needed( + ctx: typer.Context, scan_type: str, documents_to_scan: list[Document], is_git_diff: bool = False +) -> None: + no_restore = ctx.obj.get('no_restore', False) + if scan_type == consts.SCA_SCAN_TYPE and not no_restore: + _add_dependencies_tree_documents(ctx, documents_to_scan, is_git_diff) diff --git a/cycode/cli/files_collector/walk_ignore.py b/cycode/cli/files_collector/walk_ignore.py new file mode 100644 index 00000000..0c9d53a3 --- /dev/null +++ b/cycode/cli/files_collector/walk_ignore.py @@ -0,0 +1,68 @@ +import os +from collections.abc import Generator, Iterable + +from cycode.cli import consts +from cycode.cli.logger import get_logger +from cycode.cli.utils.ignore_utils import IgnoreFilterManager + +logger = get_logger('Ignores') + +_SUPPORTED_IGNORE_PATTERN_FILES = { + '.gitignore', +} +_DEFAULT_GLOBAL_IGNORE_PATTERNS = [ + '.git', + '.cycode', +] + + +def _walk_to_top(path: str) -> Iterable[str]: + while os.path.dirname(path) != path: + yield path + path = os.path.dirname(path) + + if path: + yield path # Include the top-level directory + + +def _collect_top_level_ignore_files(path: str, *, is_cycodeignore_allowed: bool = True) -> list[str]: + ignore_files = [] + top_paths = reversed(list(_walk_to_top(path))) # we must reverse it to make top levels more prioritized + + supported_files = set(_SUPPORTED_IGNORE_PATTERN_FILES) + if is_cycodeignore_allowed: + supported_files.add(consts.CYCODEIGNORE_FILENAME) + logger.debug('.cycodeignore files included due to scan configuration') + + for dir_path in top_paths: + for ignore_file in supported_files: + ignore_file_path = os.path.join(dir_path, ignore_file) + if os.path.exists(ignore_file_path): + logger.debug('Reading top level ignore file: %s', ignore_file_path) + ignore_files.append(ignore_file_path) + return ignore_files + + +def walk_ignore( + path: str, *, is_cycodeignore_allowed: bool = True +) -> Generator[tuple[str, list[str], list[str]], None, None]: + ignore_file_paths = _collect_top_level_ignore_files(path, is_cycodeignore_allowed=is_cycodeignore_allowed) + ignore_filter_manager = IgnoreFilterManager.build( + path=path, + global_ignore_file_paths=ignore_file_paths, + global_patterns=_DEFAULT_GLOBAL_IGNORE_PATTERNS, + ) + for dirpath, dirnames, filenames, ignored_dirnames, ignored_filenames in ignore_filter_manager.walk_with_ignored(): + rel_dirpath = '' if dirpath == path else os.path.relpath(dirpath, path) + display_dir = rel_dirpath or '.' + for is_dir, names in ( + (True, ignored_dirnames), + (False, ignored_filenames), + ): + for name in names: + full_path = os.path.join(path, display_dir, name) + if is_dir: + full_path = os.path.join(full_path, '*') + logger.debug('Ignoring match %s', full_path) + + yield dirpath, dirnames, filenames diff --git a/cycode/cli/files_collector/zip_documents.py b/cycode/cli/files_collector/zip_documents.py new file mode 100644 index 00000000..7927bdc6 --- /dev/null +++ b/cycode/cli/files_collector/zip_documents.py @@ -0,0 +1,53 @@ +import timeit +from pathlib import Path +from typing import Optional + +from cycode.cli import consts +from cycode.cli.exceptions import custom_exceptions +from cycode.cli.files_collector.models.in_memory_zip import InMemoryZip +from cycode.cli.models import Document +from cycode.logger import get_logger + +logger = get_logger('ZIP') + + +def _validate_zip_file_size(scan_type: str, zip_file_size: int) -> None: + max_size_limit = consts.ZIP_MAX_SIZE_LIMIT_IN_BYTES.get(scan_type, consts.DEFAULT_ZIP_MAX_SIZE_LIMIT_IN_BYTES) + if zip_file_size > max_size_limit: + raise custom_exceptions.ZipTooLargeError(max_size_limit) + + +def zip_documents( + scan_type: str, + documents: list[Document], + zip_file: Optional[InMemoryZip] = None, +) -> InMemoryZip: + if zip_file is None: + zip_file = InMemoryZip() + + start_zip_creation_time = timeit.default_timer() + + for index, document in enumerate(documents): + _validate_zip_file_size(scan_type, zip_file.size) + + logger.debug( + 'Adding file to ZIP, %s', + {'index': index, 'filename': document.path, 'unique_id': document.unique_id}, + ) + zip_file.append(document.path, document.unique_id, document.content) + + zip_file.close() + + end_zip_creation_time = timeit.default_timer() + zip_creation_time = int(end_zip_creation_time - start_zip_creation_time) + logger.debug( + 'Finished to create ZIP file, %s', + {'zip_creation_time': zip_creation_time, 'zip_size': zip_file.size, 'documents_count': len(documents)}, + ) + + if zip_file.configuration_manager.get_debug_flag(): + zip_file_path = Path.joinpath(Path.cwd(), f'{scan_type}_scan_{end_zip_creation_time}.zip') + logger.debug('Writing ZIP file to disk, %s', {'zip_file_path': zip_file_path}) + zip_file.write_on_disk(zip_file_path) + + return zip_file diff --git a/cycode/cli/helpers/maven/base_restore_maven_dependencies.py b/cycode/cli/helpers/maven/base_restore_maven_dependencies.py deleted file mode 100644 index 484df810..00000000 --- a/cycode/cli/helpers/maven/base_restore_maven_dependencies.py +++ /dev/null @@ -1,61 +0,0 @@ -from abc import ABC, abstractmethod -from typing import List, Optional, Dict - -import click - -from cycode.cli.models import Document -from cycode.cli.utils.path_utils import join_paths, get_file_dir -from cycode.cli.utils.shell_executor import shell -from cycode.cyclient import logger - - -def build_dep_tree_path(path: str, generated_file_name: str) -> str: - return join_paths(get_file_dir(path), generated_file_name) - - -def execute_command(command: List[str], file_name: str, command_timeout: int) -> Optional[Dict]: - try: - dependencies = shell(command, command_timeout) - except Exception as e: - logger.debug('Failed to restore dependencies shell comment. %s', - {'filename': file_name, 'exception': str(e)}) - return None - - return dependencies - - -class BaseRestoreMavenDependencies(ABC): - - def __init__(self, context: click.Context, is_git_diff: bool, - command_timeout: int): - self.context = context - self.is_git_diff = is_git_diff - self.command_timeout = command_timeout - - def restore(self, document: Document) -> Optional[Document]: - restore_dependencies_document = self.try_restore_dependencies(document) - return restore_dependencies_document - - def get_manifest_file_path(self, document: Document) -> str: - return join_paths(self.context.params.get('path'), document.path) if self.context.obj.get( - 'monitor') else document.path - - @abstractmethod - def is_project(self, document: Document) -> bool: - pass - - @abstractmethod - def get_command(self, manifest_file_path: str) -> List[str]: - pass - - @abstractmethod - def get_lock_file_name(self) -> str: - pass - - def try_restore_dependencies(self, document: Document) -> Optional[Document]: - manifest_file_path = self.get_manifest_file_path(document) - document = Document(build_dep_tree_path(document.path, self.get_lock_file_name()), - execute_command(self.get_command(manifest_file_path), manifest_file_path, - self.command_timeout), - self.is_git_diff) - return document diff --git a/cycode/cli/helpers/maven/restore_gradle_dependencies.py b/cycode/cli/helpers/maven/restore_gradle_dependencies.py deleted file mode 100644 index 1e61ee7f..00000000 --- a/cycode/cli/helpers/maven/restore_gradle_dependencies.py +++ /dev/null @@ -1,25 +0,0 @@ -from typing import List - -import click - -from cycode.cli.helpers.maven.base_restore_maven_dependencies import BaseRestoreMavenDependencies -from cycode.cli.models import Document - -BUILD_GRADLE_FILE_NAME = 'build.gradle' -BUILD_GRADLE_KTS_FILE_NAME = 'build.gradle.kts' -BUILD_GRADLE_DEP_TREE_FILE_NAME = 'gradle-dependencies-generated.txt' - - -class RestoreGradleDependencies(BaseRestoreMavenDependencies): - def __init__(self, context: click.Context, is_git_diff: bool, - command_timeout: int): - super().__init__(context, is_git_diff, command_timeout) - - def is_project(self, document: Document) -> bool: - return document.path.endswith(BUILD_GRADLE_FILE_NAME) or document.path.endswith(BUILD_GRADLE_KTS_FILE_NAME) - - def get_command(self, manifest_file_path: str) -> List[str]: - return ['gradle', 'dependencies', '-b', manifest_file_path, '-q', '--console', 'plain'] - - def get_lock_file_name(self) -> str: - return BUILD_GRADLE_DEP_TREE_FILE_NAME diff --git a/cycode/cli/helpers/maven/restore_maven_dependencies.py b/cycode/cli/helpers/maven/restore_maven_dependencies.py deleted file mode 100644 index 8ab21ca3..00000000 --- a/cycode/cli/helpers/maven/restore_maven_dependencies.py +++ /dev/null @@ -1,62 +0,0 @@ -from os import path -from typing import List, Optional - -import click - -from cycode.cli.helpers.maven.base_restore_maven_dependencies import BaseRestoreMavenDependencies, build_dep_tree_path, \ - execute_command -from cycode.cli.models import Document -from cycode.cli.utils.path_utils import get_file_dir, get_file_content, join_paths - -BUILD_MAVEN_FILE_NAME = 'pom.xml' -MAVEN_CYCLONE_DEP_TREE_FILE_NAME = 'bom.json' -MAVEN_DEP_TREE_FILE_NAME = 'bcde.mvndeps' - - -class RestoreMavenDependencies(BaseRestoreMavenDependencies): - def __init__(self, context: click.Context, is_git_diff: bool, - command_timeout: int): - super().__init__(context, is_git_diff, command_timeout) - - def is_project(self, document: Document) -> bool: - return path.basename(document.path).split('/')[-1] == BUILD_MAVEN_FILE_NAME - - def get_command(self, manifest_file_path: str) -> List[str]: - return ['mvn', 'org.cyclonedx:cyclonedx-maven-plugin:2.7.4:makeAggregateBom', '-f', manifest_file_path] - - def get_lock_file_name(self) -> str: - return join_paths('target', MAVEN_CYCLONE_DEP_TREE_FILE_NAME) - - def try_restore_dependencies(self, document: Document) -> Optional[Document]: - restore_dependencies_document = super().try_restore_dependencies(document) - manifest_file_path = self.get_manifest_file_path(document) - if document.content is None: - restore_dependencies_document = self.restore_from_secondary_command(document, manifest_file_path, - restore_dependencies_document) - else: - restore_dependencies_document.content = get_file_content( - join_paths(get_file_dir(manifest_file_path), self.get_lock_file_name())) - - return restore_dependencies_document - - def restore_from_secondary_command(self, document, manifest_file_path, restore_dependencies_document): - # TODO(MarshalX): does it even work? Missing argument - secondary_restore_command = create_secondary_restore_command(manifest_file_path) - backup_restore_content = execute_command( - secondary_restore_command, - manifest_file_path, - self.command_timeout) - restore_dependencies_document = Document(build_dep_tree_path(document.path, MAVEN_DEP_TREE_FILE_NAME), - backup_restore_content, - self.is_git_diff) - restore_dependencies = None - if restore_dependencies_document.content is not None: - restore_dependencies = restore_dependencies_document - restore_dependencies.content = get_file_content(MAVEN_DEP_TREE_FILE_NAME) - - return restore_dependencies - - -def create_secondary_restore_command(self, manifest_file_path): - return ['mvn', 'dependency:tree', '-B', '-DoutputType=text', '-f', manifest_file_path, - f'-DoutputFile={MAVEN_DEP_TREE_FILE_NAME}'] diff --git a/cycode/cli/helpers/sca_code_scanner.py b/cycode/cli/helpers/sca_code_scanner.py deleted file mode 100644 index e12342fd..00000000 --- a/cycode/cli/helpers/sca_code_scanner.py +++ /dev/null @@ -1,125 +0,0 @@ -import os -from typing import List, Optional, Dict - -import click -from git import Repo, GitCommandError - -from cycode.cli.consts import * -from cycode.cli.helpers.maven.restore_gradle_dependencies import RestoreGradleDependencies -from cycode.cli.helpers.maven.restore_maven_dependencies import RestoreMavenDependencies -from cycode.cli.models import Document -from cycode.cli.utils.path_utils import get_file_dir, join_paths, get_file_content -from cycode.cyclient import logger - -BUILD_GRADLE_FILE_NAME = 'build.gradle' -BUILD_GRADLE_KTS_FILE_NAME = 'build.gradle.kts' -BUILD_GRADLE_DEP_TREE_FILE_NAME = 'gradle-dependencies-generated.txt' -BUILD_GRADLE_DEP_TREE_TIMEOUT = 180 - - -def perform_pre_commit_range_scan_actions(path: str, from_commit_documents: List[Document], - from_commit_rev: str, to_commit_documents: List[Document], - to_commit_rev: str) -> None: - repo = Repo(path) - add_ecosystem_related_files_if_exists(from_commit_documents, repo, from_commit_rev) - add_ecosystem_related_files_if_exists(to_commit_documents, repo, to_commit_rev) - - -def perform_pre_hook_range_scan_actions(git_head_documents: List[Document], - pre_committed_documents: List[Document]) -> None: - repo = Repo(os.getcwd()) - add_ecosystem_related_files_if_exists(git_head_documents, repo, GIT_HEAD_COMMIT_REV) - add_ecosystem_related_files_if_exists(pre_committed_documents) - - -def add_ecosystem_related_files_if_exists(documents: List[Document], repo: Optional[Repo] = None, - commit_rev: Optional[str] = None): - for doc in documents: - ecosystem = get_project_file_ecosystem(doc) - if ecosystem is None: - logger.debug("failed to resolve project file ecosystem: %s", doc.path) - continue - documents_to_add = get_doc_ecosystem_related_project_files(doc, documents, ecosystem, commit_rev, repo) - documents.extend(documents_to_add) - - -def get_doc_ecosystem_related_project_files(doc: Document, documents: List[Document], ecosystem: str, - commit_rev: Optional[str], repo: Optional[Repo]) -> List[Document]: - documents_to_add: List[Document] = [] - for ecosystem_project_file in PROJECT_FILES_BY_ECOSYSTEM_MAP.get(ecosystem): - file_to_search = join_paths(get_file_dir(doc.path), ecosystem_project_file) - if not is_project_file_exists_in_documents(documents, file_to_search): - file_content = get_file_content_from_commit(repo, commit_rev, file_to_search) if repo \ - else get_file_content(file_to_search) - - if file_content is not None: - documents_to_add.append(Document(file_to_search, file_content)) - - return documents_to_add - - -def is_project_file_exists_in_documents(documents: List[Document], file: str) -> bool: - return any(doc for doc in documents if file == doc.path) - - -def get_project_file_ecosystem(document: Document) -> Optional[str]: - for ecosystem, project_files in PROJECT_FILES_BY_ECOSYSTEM_MAP.items(): - for project_file in project_files: - if document.path.endswith(project_file): - return ecosystem - return None - - -def try_restore_dependencies(context: click.Context, documents_to_add: List[Document], restore_dependencies, - document: Document): - if restore_dependencies.is_project(document): - restore_dependencies_document = restore_dependencies.restore(document) - if restore_dependencies_document is None: - logger.warning('Error occurred while trying to generate dependencies tree. %s', - {'filename': document.path}) - return - - if restore_dependencies_document.content is None: - logger.warning('Error occurred while trying to generate dependencies tree. %s', - {'filename': document.path}) - restore_dependencies_document.content = '' - else: - is_monitor_action = context.obj.get('monitor') - project_path = context.params.get('path') - manifest_file_path = get_manifest_file_path(document, is_monitor_action, project_path) - logger.debug(f"Succeeded to generate dependencies tree on path: {manifest_file_path}") - - if restore_dependencies_document.path in documents_to_add: - logger.debug(f"Duplicate document on restore for path: {restore_dependencies_document.path}") - else: - documents_to_add[restore_dependencies_document.path] = restore_dependencies_document - - -def add_dependencies_tree_document(context: click.Context, documents_to_scan: List[Document], - is_git_diff: bool = False) -> None: - documents_to_add: Dict[str, Document] = {} - restore_dependencies_list = restore_handlers(context, is_git_diff) - - for restore_dependencies in restore_dependencies_list: - for document in documents_to_scan: - try_restore_dependencies(context, documents_to_add, restore_dependencies, document) - - documents_to_scan.extend(list(documents_to_add.values())) - - -def restore_handlers(context, is_git_diff): - return [ - RestoreGradleDependencies(context, is_git_diff, BUILD_GRADLE_DEP_TREE_TIMEOUT), - RestoreMavenDependencies(context, is_git_diff, BUILD_GRADLE_DEP_TREE_TIMEOUT) - ] - - -def get_manifest_file_path(document: Document, is_monitor_action: bool, project_path: str) -> str: - return join_paths(project_path, document.path) if is_monitor_action else document.path - - -def get_file_content_from_commit(repo: Repo, commit: str, file_path: str) -> Optional[str]: - try: - return repo.git.show(f'{commit}:{file_path}') - except GitCommandError: - return None diff --git a/cycode/cli/logger.py b/cycode/cli/logger.py new file mode 100644 index 00000000..46748bff --- /dev/null +++ b/cycode/cli/logger.py @@ -0,0 +1,3 @@ +from cycode.logger import get_logger + +logger = get_logger('CLI') diff --git a/cycode/cli/main.py b/cycode/cli/main.py index 9eee8343..c6a857a4 100644 --- a/cycode/cli/main.py +++ b/cycode/cli/main.py @@ -1,198 +1,10 @@ -import logging +from multiprocessing import freeze_support -import click -import sys +from cycode.cli.app import app -from typing import List, Optional +# DO NOT REMOVE OR MOVE THIS LINE +# this is required to support multiprocessing in executables files packaged with PyInstaller +# see https://pyinstaller.org/en/latest/common-issues-and-pitfalls.html#multi-processing +freeze_support() -from cycode import __version__ -from cycode.cli.models import Severity -from cycode.cli.config import config -from cycode.cli import code_scanner -from cycode.cli.user_settings.credentials_manager import CredentialsManager -from cycode.cli.user_settings.configuration_manager import ConfigurationManager -from cycode.cli.user_settings.user_settings_commands import set_credentials, add_exclusions -from cycode.cli.auth.auth_command import authenticate -from cycode.cli.utils import scan_utils -from cycode.cyclient import logger -from cycode.cyclient.cycode_client_base import CycodeClientBase -from cycode.cyclient.models import UserAgentOptionScheme -from cycode.cyclient.scan_config.scan_config_creator import create_scan_client - -CONTEXT = dict() -ISSUE_DETECTED_STATUS_CODE = 1 -NO_ISSUES_STATUS_CODE = 0 - - -@click.group( - commands={ - "repository": code_scanner.scan_repository, - "commit_history": code_scanner.scan_repository_commit_history, - "path": code_scanner.scan_path, - "pre_commit": code_scanner.pre_commit_scan, - "pre_receive": code_scanner.pre_receive_scan - }, -) -@click.option('--scan-type', '-t', default="secret", - help=""" - \b - Specify the scan you wish to execute (secret/iac/sca), - the default is secret - """, - type=click.Choice(config['scans']['supported_scans'])) -@click.option('--secret', - default=None, - help='Specify a Cycode client secret for this specific scan execution', - type=str, - required=False) -@click.option('--client-id', - default=None, - help='Specify a Cycode client ID for this specific scan execution', - type=str, - required=False) -@click.option('--show-secret', - is_flag=True, - default=False, - help='Show secrets in plain text', - type=bool, - required=False) -@click.option('--soft-fail', - is_flag=True, - default=False, - help='Run scan without failing, always return a non-error status code', - type=bool, - required=False) -@click.option('--output', default=None, - help=""" - \b - Specify the results output (text/json/table), - the default is text - """, - type=click.Choice(['text', 'json', 'table'])) -@click.option('--severity-threshold', - default=None, - help='Show only violations at the specified level or higher (supported for SCA scan type only).', - type=click.Choice([e.name for e in Severity]), - required=False) -@click.option('--sca-scan', - default=None, - help="Specify the sca scan you wish to execute (package-vulnerabilities/license-compliance), the default is both", - multiple=True, - type=click.Choice(config['scans']['supported_sca_scans'])) -@click.option('--monitor', - is_flag=True, - default=False, - help="When specified, the scan results will be recorded in the knowledge graph. Please note that when working in 'monitor' mode, the knowledge graph will not be updated as a result of SCM events (Push, Repo creation).(supported for SCA scan type only).", - type=bool, - required=False) -@click.option('--report', - is_flag=True, - default=False, - help="When specified, a violations report will be generated. A URL link to the report will be printed as an output to the command execution", - type=bool, - required=False) -@click.pass_context -def code_scan(context: click.Context, scan_type, client_id, secret, show_secret, soft_fail, output, severity_threshold, - sca_scan: List[str], monitor, report): - """ Scan content for secrets/IaC/sca/SAST violations, You need to specify which scan type: ci/commit_history/path/repository/etc """ - if show_secret: - context.obj["show_secret"] = show_secret - else: - context.obj["show_secret"] = config["result_printer"]["default"]["show_secret"] - - if soft_fail: - context.obj["soft_fail"] = soft_fail - else: - context.obj["soft_fail"] = config["soft_fail"] - - context.obj["scan_type"] = scan_type - if output is not None: - # save backward compatability with old style command - context.obj["output"] = output - context.obj["client"] = get_cycode_client(client_id, secret) - context.obj["severity_threshold"] = severity_threshold - context.obj["monitor"] = monitor - context.obj["report"] = report - _sca_scan_to_context(context, sca_scan) - - return 1 - - -@code_scan.result_callback() -@click.pass_context -def finalize(context: click.Context, *args, **kwargs): - if context.obj["soft_fail"]: - sys.exit(0) - - sys.exit(ISSUE_DETECTED_STATUS_CODE if _should_fail_scan(context) else NO_ISSUES_STATUS_CODE) - - -@click.group( - commands={ - "scan": code_scan, - "configure": set_credentials, - "ignore": add_exclusions, - "auth": authenticate - }, - context_settings=CONTEXT -) -@click.option( - "--verbose", "-v", is_flag=True, default=False, help="Show detailed logs", -) -@click.option( - '--output', - default='text', - help='Specify the output (text/json/table), the default is text', - type=click.Choice(['text', 'json', 'table']) -) -@click.option( - '--user-agent', - default=None, - help='Characteristic JSON object that lets servers identify the application', - type=str, -) -@click.version_option(__version__, prog_name="cycode") -@click.pass_context -def main_cli(context: click.Context, verbose: bool, output: str, user_agent: Optional[str]): - context.ensure_object(dict) - configuration_manager = ConfigurationManager() - - verbose = verbose or configuration_manager.get_verbose_flag() - context.obj['verbose'] = verbose - log_level = logging.DEBUG if verbose else logging.INFO - logger.setLevel(log_level) - - context.obj['output'] = output - - if user_agent: - user_agent_option = UserAgentOptionScheme().loads(user_agent) - CycodeClientBase.enrich_user_agent(user_agent_option.user_agent_suffix) - - -def get_cycode_client(client_id, client_secret): - if not client_id or not client_secret: - client_id, client_secret = _get_configured_credentials() - if not client_id: - raise click.ClickException("Cycode client id needed.") - if not client_secret: - raise click.ClickException("Cycode client secret is needed.") - - return create_scan_client(client_id, client_secret) - - -def _get_configured_credentials(): - credentials_manager = CredentialsManager() - return credentials_manager.get_credentials() - - -def _should_fail_scan(context: click.Context): - return scan_utils.is_scan_failed(context) - - -def _sca_scan_to_context(context: click.Context, sca_scan_user_selected: List[str]): - for sca_scan_option_selected in sca_scan_user_selected: - context.obj[sca_scan_option_selected] = True - - -if __name__ == '__main__': - main_cli() +app() diff --git a/cycode/cli/models.py b/cycode/cli/models.py index 011ffe4e..3c59eeee 100644 --- a/cycode/cli/models.py +++ b/cycode/cli/models.py @@ -1,49 +1,35 @@ -from enum import Enum -from typing import List, NamedTuple, Dict, Type +from dataclasses import dataclass +from typing import NamedTuple, Optional from cycode.cyclient.models import Detection class Document: - def __init__(self, path: str, content: str, is_git_diff_format: bool = False, unique_id: str = None): + def __init__( + self, + path: str, + content: str, + is_git_diff_format: bool = False, + unique_id: Optional[str] = None, + absolute_path: Optional[str] = None, + ) -> None: self.path = path self.content = content self.is_git_diff_format = is_git_diff_format self.unique_id = unique_id + self.absolute_path = absolute_path def __repr__(self) -> str: - return ( - "path:{0}, " - "content:{1}".format(self.path, self.content) - ) + return f'path:{self.path}, content:{self.content}' class DocumentDetections: - def __init__(self, document: Document, detections: List[Detection]): + def __init__(self, document: Document, detections: list[Detection]) -> None: self.document = document self.detections = detections def __repr__(self) -> str: - return ( - "document:{0}, " - "detections:{1}".format(self.document, self.detections) - ) - - -class Severity(Enum): - INFO = -1 - LOW = 0 - MEDIUM = 1 - MODERATE = 1 - HIGH = 2 - CRITICAL = 3 - - @staticmethod - def try_get_value(name: str) -> any: - if name not in Severity.__members__: - return None - - return Severity[name].value + return f'document:{self.document}, detections:{self.detections}' class CliError(NamedTuple): @@ -51,10 +37,37 @@ class CliError(NamedTuple): message: str soft_fail: bool = False + def enrich(self, additional_message: str) -> 'CliError': + message = f'{self.message} ({additional_message})' + return CliError(self.code, message, self.soft_fail) + -CliErrors = Dict[Type[Exception], CliError] +CliErrors = dict[type[BaseException], CliError] class CliResult(NamedTuple): success: bool message: str + data: Optional[dict[str, any]] = None + + +class LocalScanResult(NamedTuple): + scan_id: str + report_url: Optional[str] + document_detections: list[DocumentDetections] + issue_detected: bool + detections_count: int + relevant_detections_count: int + + +@dataclass +class ResourceChange: + module_address: Optional[str] + resource_type: str + name: str + index: Optional[int] + actions: list[str] + values: dict[str, str] + + def __repr__(self) -> str: + return f'resource_type: {self.resource_type}, name: {self.name}' diff --git a/cycode/cli/printers/base_printer.py b/cycode/cli/printers/base_printer.py deleted file mode 100644 index e5450e8f..00000000 --- a/cycode/cli/printers/base_printer.py +++ /dev/null @@ -1,27 +0,0 @@ -from abc import ABC, abstractmethod -from typing import List - -import click - -from cycode.cli.models import DocumentDetections, CliResult, CliError - - -class BasePrinter(ABC): - RED_COLOR_NAME = 'red' - WHITE_COLOR_NAME = 'white' - GREEN_COLOR_NAME = 'green' - - def __init__(self, context: click.Context): - self.context = context - - @abstractmethod - def print_scan_results(self, results: List[DocumentDetections]) -> None: - pass - - @abstractmethod - def print_result(self, result: CliResult) -> None: - pass - - @abstractmethod - def print_error(self, error: CliError) -> None: - pass diff --git a/cycode/cli/printers/base_table_printer.py b/cycode/cli/printers/base_table_printer.py deleted file mode 100644 index 112a1d4a..00000000 --- a/cycode/cli/printers/base_table_printer.py +++ /dev/null @@ -1,43 +0,0 @@ -import abc -from typing import List - -import click - -from cycode.cli.printers.text_printer import TextPrinter -from cycode.cli.models import DocumentDetections, CliError, CliResult -from cycode.cli.printers.base_printer import BasePrinter - - -class BaseTablePrinter(BasePrinter, abc.ABC): - def __init__(self, context: click.Context): - super().__init__(context) - self.context = context - self.scan_id: str = context.obj.get('scan_id') - self.scan_type: str = context.obj.get('scan_type') - self.show_secret: bool = context.obj.get('show_secret', False) - - def print_result(self, result: CliResult) -> None: - TextPrinter(self.context).print_result(result) - - def print_error(self, error: CliError) -> None: - TextPrinter(self.context).print_error(error) - - def print_scan_results(self, results: List[DocumentDetections]): - click.secho(f'Scan Results: (scan_id: {self.scan_id})') - - if not results: - click.secho('Good job! No issues were found!!! 👏👏👏', fg=self.GREEN_COLOR_NAME) - return - - self._print_results(results) - - report_url = self.context.obj.get('report_url') - if report_url: - click.secho(f'Report URL: {report_url}') - - def _is_git_repository(self) -> bool: - return self.context.obj.get('remote_url') is not None - - @abc.abstractmethod - def _print_results(self, results: List[DocumentDetections]) -> None: - raise NotImplementedError diff --git a/cycode/cli/printers/console_printer.py b/cycode/cli/printers/console_printer.py index 9932e140..50d48fd7 100644 --- a/cycode/cli/printers/console_printer.py +++ b/cycode/cli/printers/console_printer.py @@ -1,51 +1,159 @@ -import click -from typing import List, TYPE_CHECKING +import io +from typing import TYPE_CHECKING, ClassVar, Optional +import typer +from rich.console import Console + +from cycode.cli import consts +from cycode.cli.cli_types import ExportTypeOption +from cycode.cli.console import console, console_err from cycode.cli.exceptions.custom_exceptions import CycodeError -from cycode.cli.models import DocumentDetections, CliResult, CliError -from cycode.cli.printers.table_printer import TablePrinter -from cycode.cli.printers.sca_table_printer import SCATablePrinter +from cycode.cli.models import CliError, CliResult from cycode.cli.printers.json_printer import JsonPrinter +from cycode.cli.printers.rich_printer import RichPrinter +from cycode.cli.printers.tables.sca_table_printer import ScaTablePrinter +from cycode.cli.printers.tables.table_printer import TablePrinter from cycode.cli.printers.text_printer import TextPrinter if TYPE_CHECKING: - from cycode.cli.printers.base_printer import BasePrinter + from pathlib import Path + + from cycode.cli.models import LocalScanResult + from cycode.cli.printers.tables.table_printer_base import PrinterBase class ConsolePrinter: - _AVAILABLE_PRINTERS = { + _AVAILABLE_PRINTERS: ClassVar[dict[str, type['PrinterBase']]] = { + 'rich': RichPrinter, 'text': TextPrinter, 'json': JsonPrinter, 'table': TablePrinter, - # overrides - 'table_sca': SCATablePrinter, - 'text_sca': SCATablePrinter, + # overrides: + 'table_sca': ScaTablePrinter, } - def __init__(self, context: click.Context): - self.context = context - self.scan_type = self.context.obj.get('scan_type') - self.output_type = self.context.obj.get('output') + def __init__( + self, + ctx: typer.Context, + console_override: Optional['Console'] = None, + console_err_override: Optional['Console'] = None, + output_type_override: Optional[str] = None, + ) -> None: + self.ctx = ctx + self.console = console_override or console + self.console_err = console_err_override or console_err + self.output_type = output_type_override or self.ctx.obj.get('output') - self._printer_class = self._AVAILABLE_PRINTERS.get(self.output_type) - if self._printer_class is None: - raise CycodeError(f'"{self.output_type}" output type is not supported.') + self.export_type: Optional[str] = None + self.export_file: Optional[Path] = None + self.console_record: Optional[ConsolePrinter] = None - def print_scan_results(self, detections_results_list: List[DocumentDetections]) -> None: - printer = self._get_scan_printer() - printer.print_scan_results(detections_results_list) + @property + def scan_type(self) -> str: + return self.ctx.obj.get('scan_type') - def _get_scan_printer(self) -> 'BasePrinter': - printer_class = self._printer_class + @property + def aggregation_report_url(self) -> str: + return self.ctx.obj.get('aggregation_report_url') + + @property + def printer(self) -> 'PrinterBase': + printer_class = self._AVAILABLE_PRINTERS.get(self.output_type) composite_printer = self._AVAILABLE_PRINTERS.get(f'{self.output_type}_{self.scan_type}') if composite_printer: printer_class = composite_printer - return printer_class(self.context) + if not printer_class: + raise CycodeError(f'"{self.output_type}" output type is not supported.') + + return printer_class(self.ctx, self.console, self.console_err) + + def update_ctx(self, ctx: 'typer.Context') -> None: + self.ctx = ctx + + def enable_recording(self, export_type: str, export_file: 'Path') -> None: + if self.console_record is None: + self.export_file = export_file + self.export_type = export_type + + self.console_record = ConsolePrinter( + self.ctx, + console_override=Console(record=True, file=io.StringIO()), + console_err_override=Console(stderr=True, record=True, file=io.StringIO()), + output_type_override='json' if self.export_type == 'json' else self.output_type, + ) + + def print_scan_results( + self, + local_scan_results: list['LocalScanResult'], + errors: Optional[dict[str, 'CliError']] = None, + ) -> None: + if self.console_record: + self.console_record.print_scan_results(local_scan_results, errors) + self.printer.print_scan_results(local_scan_results, errors) def print_result(self, result: CliResult) -> None: - self._printer_class(self.context).print_result(result) + if self.console_record: + self.console_record.print_result(result) + self.printer.print_result(result) def print_error(self, error: CliError) -> None: - self._printer_class(self.context).print_error(error) + if self.console_record: + self.console_record.print_error(error) + self.printer.print_error(error) + + def print_exception(self, e: Optional[BaseException] = None, force_print: bool = False) -> None: + """Print traceback message in stderr if verbose mode is set.""" + if force_print or self.ctx.obj.get('verbose', False): + if self.console_record: + self.console_record.print_exception(e) + self.printer.print_exception(e) + + def export(self) -> None: + if self.console_record is None: + raise CycodeError('Console recording was not enabled. Cannot export.') + + export_file = self.export_file + if not export_file.suffix: + # resolve file extension based on the export type if not provided in the file name + export_file = export_file.with_suffix(f'.{self.export_type.lower()}') + + export_file = str(export_file) + if self.export_type is ExportTypeOption.HTML: + self.console_record.console.save_html(export_file) + elif self.export_type is ExportTypeOption.SVG: + self.console_record.console.save_svg(export_file, title=consts.APP_NAME) + elif self.export_type is ExportTypeOption.JSON: + with open(export_file, 'w', encoding='UTF-8') as f: + self.console_record.console.file.seek(0) + f.write(self.console_record.console.file.read()) + else: + raise CycodeError(f'Export type "{self.export_type}" is not supported.') + + export_format_msg = f'{self.export_type.upper()} format' + if self.export_type in {ExportTypeOption.HTML, ExportTypeOption.SVG}: + export_format_msg += f' with {self.output_type.upper()} output type' + + clickable_path = f'[link=file://{self.export_file}]{self.export_file}[/link]' + self.console.print(f'[b green]Cycode CLI output exported to {clickable_path} in {export_format_msg}[/]') + + @property + def is_recording(self) -> bool: + return self.console_record is not None + + @property + def is_json_printer(self) -> bool: + return isinstance(self.printer, JsonPrinter) + + @property + def is_table_printer(self) -> bool: + return isinstance(self.printer, TablePrinter) + + @property + def is_text_printer(self) -> bool: + return isinstance(self.printer, TextPrinter) + + @property + def is_rich_printer(self) -> bool: + return isinstance(self.printer, RichPrinter) diff --git a/cycode/cli/printers/json_printer.py b/cycode/cli/printers/json_printer.py index 2e77aba7..acb7912f 100644 --- a/cycode/cli/printers/json_printer.py +++ b/cycode/cli/printers/json_printer.py @@ -1,47 +1,60 @@ import json -from typing import List +from typing import TYPE_CHECKING, Optional -import click - -from cycode.cli.models import DocumentDetections, CliResult, CliError -from cycode.cli.printers.base_printer import BasePrinter +from cycode.cli.models import CliError, CliResult +from cycode.cli.printers.printer_base import PrinterBase from cycode.cyclient.models import DetectionSchema +if TYPE_CHECKING: + from cycode.cli.models import LocalScanResult -class JsonPrinter(BasePrinter): - def __init__(self, context: click.Context): - super().__init__(context) - self.scan_id = context.obj.get('scan_id') +class JsonPrinter(PrinterBase): def print_result(self, result: CliResult) -> None: - result = { - 'result': result.success, - 'message': result.message - } + result = {'result': result.success, 'message': result.message, 'data': result.data} - click.secho(self.get_data_json(result)) + self.console.print_json(self.get_data_json(result)) def print_error(self, error: CliError) -> None: - result = { - 'error': error.code, - 'message': error.message - } + result = {'error': error.code, 'message': error.message} - click.secho(self.get_data_json(result)) + self.console.print_json(self.get_data_json(result)) - def print_scan_results(self, results: List[DocumentDetections]) -> None: + def print_scan_results( + self, local_scan_results: list['LocalScanResult'], errors: Optional[dict[str, 'CliError']] = None + ) -> None: + scan_ids = [] + report_urls = [] detections = [] - for result in results: - detections.extend(result.detections) + aggregation_report_url = self.ctx.obj.get('aggregation_report_url') + if aggregation_report_url: + report_urls.append(aggregation_report_url) + + for local_scan_result in local_scan_results: + scan_ids.append(local_scan_result.scan_id) + + if not aggregation_report_url and local_scan_result.report_url: + report_urls.append(local_scan_result.report_url) + for document_detections in local_scan_result.document_detections: + detections.extend(document_detections.detections) detections_dict = DetectionSchema(many=True).dump(detections) - click.secho(self._get_json_scan_result(detections_dict)) + inlined_errors = [] + if errors: + # FIXME(MarshalX): we don't care about scan IDs in JSON output due to clumsy JSON root structure + inlined_errors = [err._asdict() for err in errors.values()] + + self.console.print_json(self._get_json_scan_result(scan_ids, detections_dict, report_urls, inlined_errors)) - def _get_json_scan_result(self, detections: dict) -> str: + def _get_json_scan_result( + self, scan_ids: list[str], detections: dict, report_urls: list[str], errors: list[dict] + ) -> str: result = { - 'scan_id': str(self.scan_id), - 'detections': detections + 'scan_ids': scan_ids, + 'detections': detections, + 'report_urls': report_urls, + 'errors': errors, } return self.get_data_json(result) @@ -49,4 +62,4 @@ def _get_json_scan_result(self, detections: dict) -> str: @staticmethod def get_data_json(data: dict) -> str: # ensure_ascii is disabled for symbols like "`". Eg: `cycode scan` - return json.dumps(data, indent=4, ensure_ascii=False) + return json.dumps(data, ensure_ascii=False) diff --git a/cycode/cli/printers/printer_base.py b/cycode/cli/printers/printer_base.py new file mode 100644 index 00000000..69596e2a --- /dev/null +++ b/cycode/cli/printers/printer_base.py @@ -0,0 +1,115 @@ +import sys +from abc import ABC, abstractmethod +from collections import defaultdict +from typing import TYPE_CHECKING, Optional + +import typer + +from cycode.cli.cli_types import SeverityOption +from cycode.cli.models import CliError, CliResult +from cycode.cyclient.headers import get_correlation_id + +if TYPE_CHECKING: + from rich.console import Console + + from cycode.cli.models import LocalScanResult + + +from rich.traceback import Traceback as RichTraceback + + +class PrinterBase(ABC): + NO_DETECTIONS_MESSAGE = ( + '[b green]Good job! No issues were found!!! :clapping_hands::clapping_hands::clapping_hands:[/]' + ) + FAILED_SCAN_MESSAGE = ( + '[b red]Unfortunately, Cycode was unable to complete the full scan. ' + 'Please note that not all results may be available:[/]' + ) + + def __init__( + self, + ctx: typer.Context, + console: 'Console', + console_err: 'Console', + ) -> None: + self.ctx = ctx + self.console = console + self.console_err = console_err + + @property + def scan_type(self) -> str: + return self.ctx.obj.get('scan_type') + + @property + def command_scan_type(self) -> str: + return self.ctx.info_name + + @property + def show_secret(self) -> bool: + return self.ctx.obj.get('show_secret', False) + + @abstractmethod + def print_scan_results( + self, local_scan_results: list['LocalScanResult'], errors: Optional[dict[str, 'CliError']] = None + ) -> None: + pass + + @abstractmethod + def print_result(self, result: CliResult) -> None: + pass + + @abstractmethod + def print_error(self, error: CliError) -> None: + pass + + def print_exception(self, e: Optional[BaseException] = None) -> None: + """We are printing it in stderr so, we don't care about supporting JSON and TABLE outputs. + + Note: + Called only when the verbose flag is set. + + """ + rich_traceback = ( + RichTraceback.from_exception(type(e), e, e.__traceback__) + if e + else RichTraceback.from_exception(*sys.exc_info()) + ) + rich_traceback.show_locals = False + self.console_err.print(rich_traceback) + + self.console_err.print(f'[red]Correlation ID:[/] {get_correlation_id()}') + + def print_scan_results_summary(self, local_scan_results: list['LocalScanResult']) -> None: + """Print a summary of scan results based on severity levels. + + Args: + local_scan_results (List['LocalScanResult']): A list of local scan results containing detections. + + The summary includes the count of detections for each severity level + and is displayed in the console in a formatted string. + + """ + detections_count = 0 + severity_counts = defaultdict(int) + for local_scan_result in local_scan_results: + for document_detections in local_scan_result.document_detections: + for detection in document_detections.detections: + if detection.severity: + detections_count += 1 + severity_counts[SeverityOption(detection.severity)] += 1 + + self.console.line() + self.console.print(f'[bold]Cycode found {detections_count} violations[/]', end=': ') + + # Example of string: CRITICAL - 6 | HIGH - 0 | MEDIUM - 14 | LOW - 0 | INFO - 0 + for index, severity in enumerate(reversed(SeverityOption), start=1): + end = ' | ' + if index == len(SeverityOption): + end = '\n' + + self.console.print( + SeverityOption.get_member_emoji(severity), severity, '-', severity_counts[severity], end=end + ) + + self.console.line() diff --git a/cycode/cli/printers/rich_printer.py b/cycode/cli/printers/rich_printer.py new file mode 100644 index 00000000..10cf561c --- /dev/null +++ b/cycode/cli/printers/rich_printer.py @@ -0,0 +1,177 @@ +from typing import TYPE_CHECKING, Optional + +from rich.console import Group +from rich.panel import Panel +from rich.table import Table +from rich.text import Text + +from cycode.cli import consts +from cycode.cli.cli_types import SeverityOption +from cycode.cli.printers.text_printer import TextPrinter +from cycode.cli.printers.utils.code_snippet_syntax import get_code_snippet_syntax +from cycode.cli.printers.utils.detection_data import ( + get_detection_clickable_cwe_cve, + get_detection_file_path, + get_detection_title, +) +from cycode.cli.printers.utils.detection_ordering.common_ordering import sort_and_group_detections_from_scan_result +from cycode.cli.printers.utils.rich_helpers import get_columns_in_1_to_3_ratio, get_markdown_panel, get_panel + +if TYPE_CHECKING: + from cycode.cli.models import CliError, Detection, Document, LocalScanResult + + +class RichPrinter(TextPrinter): + MAX_PATH_LENGTH = 60 + + def print_scan_results( + self, local_scan_results: list['LocalScanResult'], errors: Optional[dict[str, 'CliError']] = None + ) -> None: + if not errors and all(result.issue_detected == 0 for result in local_scan_results): + self.console.print(self.NO_DETECTIONS_MESSAGE) + return + + detections, _ = sort_and_group_detections_from_scan_result(local_scan_results) + detections_count = len(detections) + for detection_number, (detection, document) in enumerate(detections, start=1): + self._print_violation_card( + document, + detection, + detection_number, + detections_count, + ) + + self.print_scan_results_summary(local_scan_results) + self.print_report_urls_and_errors(local_scan_results, errors) + + def _get_details_table(self, detection: 'Detection') -> Table: + details_table = Table(show_header=False, box=None, padding=(0, 1)) + + details_table.add_column('Key', style='dim') + details_table.add_column('Value', style='', overflow='fold') + + severity = detection.severity if detection.severity else 'N/A' + severity_icon = SeverityOption.get_member_emoji(severity.lower()) + details_table.add_row('Severity', f'{severity_icon} {SeverityOption(severity).__rich__()}') + + path = str(get_detection_file_path(self.scan_type, detection)) + shorten_path = f'...{path[-self.MAX_PATH_LENGTH :]}' if len(path) > self.MAX_PATH_LENGTH else path + details_table.add_row('In file', f'[link=file://{path}]{shorten_path}[/]') + + self._add_scan_related_rows(details_table, detection) + + details_table.add_row('Rule ID', detection.detection_rule_id) + + return details_table + + def _add_scan_related_rows(self, details_table: Table, detection: 'Detection') -> None: + scan_type_details_handlers = { + consts.SECRET_SCAN_TYPE: self.__add_secret_scan_related_rows, + consts.SCA_SCAN_TYPE: self.__add_sca_scan_related_rows, + consts.IAC_SCAN_TYPE: self.__add_iac_scan_related_rows, + consts.SAST_SCAN_TYPE: self.__add_sast_scan_related_rows, + } + + if self.scan_type not in scan_type_details_handlers: + raise ValueError(f'Unknown scan type: {self.scan_type}') + + scan_enricher_function = scan_type_details_handlers[self.scan_type] + scan_enricher_function(details_table, detection) + + @staticmethod + def __add_secret_scan_related_rows(details_table: Table, detection: 'Detection') -> None: + details_table.add_row('Secret SHA', detection.detection_details.get('sha512')) + + @staticmethod + def __add_sca_scan_related_rows(details_table: Table, detection: 'Detection') -> None: + detection_details = detection.detection_details + + details_table.add_row('CVEs', get_detection_clickable_cwe_cve(consts.SCA_SCAN_TYPE, detection)) + details_table.add_row('Package', detection_details.get('package_name')) + details_table.add_row('Version', detection_details.get('package_version')) + + if detection.has_alert: + patched_version = detection_details['alert'].get('first_patched_version') + details_table.add_row('First patched version', patched_version or 'Not fixed') + + dependency_path = detection_details.get('dependency_paths') + details_table.add_row('Dependency path', dependency_path or 'N/A') + + if not detection.has_alert: + details_table.add_row('License', detection_details.get('license')) + + @staticmethod + def __add_iac_scan_related_rows(details_table: Table, detection: 'Detection') -> None: + details_table.add_row('IaC Provider', detection.detection_details.get('infra_provider')) + + @staticmethod + def __add_sast_scan_related_rows(details_table: Table, detection: 'Detection') -> None: + details_table.add_row('CWE', get_detection_clickable_cwe_cve(consts.SAST_SCAN_TYPE, detection)) + details_table.add_row('Subcategory', detection.detection_details.get('category')) + details_table.add_row('Language', ', '.join(detection.detection_details.get('languages', []))) + + engine_id_to_display_name = { + '5db84696-88dc-11ec-a8a3-0242ac120002': 'Semgrep OSS (Orchestrated by Cycode)', + '560a0abd-d7da-4e6d-a3f1-0ed74895295c': 'Bearer (Powered by Cycode)', + } + engine_id = detection.detection_details.get('external_scanner_id') + details_table.add_row('Security Tool', engine_id_to_display_name.get(engine_id, 'N/A')) + + def _print_violation_card( + self, document: 'Document', detection: 'Detection', detection_number: int, detections_count: int + ) -> None: + details_table = self._get_details_table(detection) + details_panel = get_panel( + details_table, + title=':mag: Details', + ) + + code_snippet_panel = get_panel( + get_code_snippet_syntax( + self.scan_type, + self.command_scan_type, + detection, + document, + obfuscate=not self.show_secret, + lines_to_display_before=3, + lines_to_display_after=3, + ), + title=':computer: Code Snippet', + ) + + if detection.has_alert: + summary = detection.detection_details['alert'].get('description') + else: + summary = detection.detection_details.get('description') or detection.message + + summary_panel = None + if summary: + summary_panel = get_markdown_panel( + summary, + title=':memo: Summary', + ) + + custom_guidelines_panel = None + custom_guidelines = detection.detection_details.get('custom_remediation_guidelines') + if custom_guidelines: + custom_guidelines_panel = get_markdown_panel( + custom_guidelines, + title=':office: Company Guidelines', + ) + + navigation = Text(f'Violation {detection_number} of {detections_count}', style='dim', justify='right') + + renderables = [navigation, get_columns_in_1_to_3_ratio(details_panel, code_snippet_panel)] + if summary_panel: + renderables.append(summary_panel) + if custom_guidelines_panel: + renderables.append(custom_guidelines_panel) + + violation_card_panel = Panel( + Group(*renderables), + title=get_detection_title(self.scan_type, detection), + border_style=SeverityOption.get_member_color(detection.severity), + title_align='center', + ) + + self.console.print(violation_card_panel) diff --git a/cycode/cli/printers/sca_table_printer.py b/cycode/cli/printers/sca_table_printer.py deleted file mode 100644 index 34130607..00000000 --- a/cycode/cli/printers/sca_table_printer.py +++ /dev/null @@ -1,142 +0,0 @@ -from collections import defaultdict -from typing import List, Dict - -import click -from texttable import Texttable - -from cycode.cli.consts import LICENSE_COMPLIANCE_POLICY_ID, PACKAGE_VULNERABILITY_POLICY_ID -from cycode.cli.models import DocumentDetections, Detection -from cycode.cli.printers.base_table_printer import BaseTablePrinter - -SEVERITY_COLUMN = 'Severity' -LICENSE_COLUMN = 'License' -UPGRADE_COLUMN = 'Upgrade' -REPOSITORY_COLUMN = 'Repository' -CVE_COLUMN = 'CVE' - -PREVIEW_DETECTIONS_COMMON_HEADERS = [ - 'File Path', - 'Ecosystem', - 'Dependency Name', - 'Direct Dependency', - 'Development Dependency' -] - - -class SCATablePrinter(BaseTablePrinter): - def _print_results(self, results: List[DocumentDetections]) -> None: - detections_per_detection_type_id = self._extract_detections_per_detection_type_id(results) - self._print_detection_per_detection_type_id(detections_per_detection_type_id) - - @staticmethod - def _extract_detections_per_detection_type_id(results: List[DocumentDetections]) -> Dict[str, List[Detection]]: - detections_per_detection_type_id = defaultdict(list) - - for document_detection in results: - for detection in document_detection.detections: - detections_per_detection_type_id[detection.detection_type_id].append(detection) - - return detections_per_detection_type_id - - def _print_detection_per_detection_type_id( - self, detections_per_detection_type_id: Dict[str, List[Detection]] - ) -> None: - for detection_type_id in detections_per_detection_type_id: - detections = detections_per_detection_type_id[detection_type_id] - headers = self._get_table_headers() - - title = None - rows = [] - - if detection_type_id == PACKAGE_VULNERABILITY_POLICY_ID: - title = "Dependencies Vulnerabilities" - - headers = [SEVERITY_COLUMN] + headers - headers.extend(PREVIEW_DETECTIONS_COMMON_HEADERS) - headers.append(CVE_COLUMN) - headers.append(UPGRADE_COLUMN) - - for detection in detections: - rows.append(self._get_upgrade_package_vulnerability(detection)) - elif detection_type_id == LICENSE_COMPLIANCE_POLICY_ID: - title = "License Compliance" - - headers.extend(PREVIEW_DETECTIONS_COMMON_HEADERS) - headers.append(LICENSE_COLUMN) - - for detection in detections: - rows.append(self._get_license(detection)) - - if rows: - self._print_table_detections(detections, headers, rows, title) - - def _get_table_headers(self) -> list: - if self._is_git_repository(): - return [REPOSITORY_COLUMN] - - return [] - - def _print_table_detections( - self, detections: List[Detection], headers: List[str], rows, title: str - ) -> None: - self._print_summary_issues(detections, title) - text_table = Texttable() - text_table.header(headers) - - self.set_table_width(headers, text_table) - - for row in rows: - text_table.add_row(row) - - click.echo(text_table.draw()) - - @staticmethod - def set_table_width(headers: List[str], text_table: Texttable) -> None: - header_width_size_cols = [] - for header in headers: - header_len = len(header) - if header == CVE_COLUMN: - header_width_size_cols.append(header_len * 5) - elif header == UPGRADE_COLUMN: - header_width_size_cols.append(header_len * 2) - else: - header_width_size_cols.append(header_len) - text_table.set_cols_width(header_width_size_cols) - - @staticmethod - def _print_summary_issues(detections: List, title: str) -> None: - click.echo(f'⛔ Found {len(detections)} issues of type: {click.style(title, bold=True)}') - - def _get_common_detection_fields(self, detection: Detection) -> List[str]: - row = [ - detection.detection_details.get('file_name'), - detection.detection_details.get('ecosystem'), - detection.detection_details.get('package_name'), - detection.detection_details.get('is_direct_dependency_str'), - detection.detection_details.get('is_dev_dependency_str') - ] - - if self._is_git_repository(): - row = [detection.detection_details.get('repository_name')] + row - - return row - - def _get_upgrade_package_vulnerability(self, detection: Detection) -> List[str]: - alert = detection.detection_details.get('alert') - row = [ - detection.detection_details.get('advisory_severity'), - *self._get_common_detection_fields(detection), - detection.detection_details.get('vulnerability_id') - ] - - upgrade = '' - if alert.get("first_patched_version"): - upgrade = f'{alert.get("vulnerable_requirements")} -> {alert.get("first_patched_version")}' - row.append(upgrade) - - return row - - def _get_license(self, detection: Detection) -> List[str]: - row = self._get_common_detection_fields(detection) - row.append(f'{detection.detection_details.get("license")}') - return row diff --git a/cycode/cli/printers/table.py b/cycode/cli/printers/table.py deleted file mode 100644 index 3677ec05..00000000 --- a/cycode/cli/printers/table.py +++ /dev/null @@ -1,61 +0,0 @@ -from typing import List, Dict, Optional, TYPE_CHECKING -from texttable import Texttable - -if TYPE_CHECKING: - from cycode.cli.printers.table_models import ColumnInfo, ColumnWidths - - -class Table: - """Helper class to manage columns and their values in the right order and only if the column should be presented.""" - - def __init__(self, column_infos: Optional[List['ColumnInfo']] = None): - self._column_widths = None - - self._columns: Dict['ColumnInfo', List[str]] = dict() - if column_infos: - self._columns: Dict['ColumnInfo', List[str]] = {columns: list() for columns in column_infos} - - def add(self, column: 'ColumnInfo') -> None: - self._columns[column] = list() - - def set(self, column: 'ColumnInfo', value: str) -> None: - # we push values only for existing columns what were added before - if column in self._columns: - self._columns[column].append(value) - - def _get_ordered_columns(self) -> List['ColumnInfo']: - # we are sorting columns by index to make sure that columns will be printed in the right order - return sorted(self._columns, key=lambda column_info: column_info.index) - - def get_columns_info(self) -> List['ColumnInfo']: - return self._get_ordered_columns() - - def get_headers(self) -> List[str]: - return [header.name for header in self._get_ordered_columns()] - - def get_rows(self) -> List[str]: - column_values = [self._columns[column_info] for column_info in self._get_ordered_columns()] - return list(zip(*column_values)) - - def set_cols_width(self, column_widths: 'ColumnWidths') -> None: - header_width_size = [] - for header in self.get_columns_info(): - width_multiplier = 1 - if header in column_widths: - width_multiplier = column_widths[header] - - header_width_size.append(len(header.name) * width_multiplier) - - self._column_widths = header_width_size - - def get_table(self, max_width: int = 80) -> Texttable: - table = Texttable(max_width) - table.header(self.get_headers()) - - for row in self.get_rows(): - table.add_row(row) - - if self._column_widths: - table.set_cols_width(self._column_widths) - - return table diff --git a/cycode/cli/printers/table_models.py b/cycode/cli/printers/table_models.py deleted file mode 100644 index 34859e06..00000000 --- a/cycode/cli/printers/table_models.py +++ /dev/null @@ -1,20 +0,0 @@ -from typing import NamedTuple, Dict - - -class ColumnInfoBuilder: - _index = 0 - - @staticmethod - def build(name: str) -> 'ColumnInfo': - column_info = ColumnInfo(name, ColumnInfoBuilder._index) - ColumnInfoBuilder._index += 1 - return column_info - - -class ColumnInfo(NamedTuple): - name: str - index: int # Represents the order of the columns, starting from the left - - -ColumnWidths = Dict[ColumnInfo, int] -ColumnWidthsConfig = Dict[str, ColumnWidths] diff --git a/cycode/cli/printers/table_printer.py b/cycode/cli/printers/table_printer.py deleted file mode 100644 index 353ce903..00000000 --- a/cycode/cli/printers/table_printer.py +++ /dev/null @@ -1,116 +0,0 @@ -from typing import List - -import click - -from cycode.cli.printers.base_table_printer import BaseTablePrinter -from cycode.cli.printers.table_models import ColumnInfoBuilder, ColumnWidthsConfig -from cycode.cli.printers.table import Table -from cycode.cli.utils.string_utils import obfuscate_text, get_position_in_line -from cycode.cli.consts import SECRET_SCAN_TYPE, INFRA_CONFIGURATION_SCAN_TYPE, SAST_SCAN_TYPE -from cycode.cli.models import DocumentDetections, Detection, Document - -# Creation must have strict order. Represents the order of the columns in the table (from left to right) -ISSUE_TYPE_COLUMN = ColumnInfoBuilder.build(name='Issue Type') -RULE_ID_COLUMN = ColumnInfoBuilder.build(name='Rule ID') -FILE_PATH_COLUMN = ColumnInfoBuilder.build(name='File Path') -SECRET_SHA_COLUMN = ColumnInfoBuilder.build(name='Secret SHA') -COMMIT_SHA_COLUMN = ColumnInfoBuilder.build(name='Commit SHA') -LINE_NUMBER_COLUMN = ColumnInfoBuilder.build(name='Line Number') -COLUMN_NUMBER_COLUMN = ColumnInfoBuilder.build(name='Column Number') -VIOLATION_LENGTH_COLUMN = ColumnInfoBuilder.build(name='Violation Length') -VIOLATION_COLUMN = ColumnInfoBuilder.build(name='Violation') - -COLUMN_WIDTHS_CONFIG: ColumnWidthsConfig = { - SECRET_SCAN_TYPE: { - ISSUE_TYPE_COLUMN: 2, - RULE_ID_COLUMN: 2, - FILE_PATH_COLUMN: 2, - SECRET_SHA_COLUMN: 2, - VIOLATION_COLUMN: 2, - }, - INFRA_CONFIGURATION_SCAN_TYPE: { - ISSUE_TYPE_COLUMN: 4, - RULE_ID_COLUMN: 3, - FILE_PATH_COLUMN: 3, - }, - SAST_SCAN_TYPE: { - ISSUE_TYPE_COLUMN: 7, - RULE_ID_COLUMN: 2, - FILE_PATH_COLUMN: 3, - }, -} - - -class TablePrinter(BaseTablePrinter): - def _print_results(self, results: List[DocumentDetections]) -> None: - table = self._get_table() - if self.scan_type in COLUMN_WIDTHS_CONFIG: - table.set_cols_width(COLUMN_WIDTHS_CONFIG[self.scan_type]) - - for result in results: - for detection in result.detections: - self._enrich_table_with_values(table, detection, result.document) - - click.echo(table.get_table().draw()) - - def _get_table(self) -> Table: - table = Table() - - table.add(ISSUE_TYPE_COLUMN) - table.add(RULE_ID_COLUMN) - table.add(FILE_PATH_COLUMN) - table.add(LINE_NUMBER_COLUMN) - table.add(COLUMN_NUMBER_COLUMN) - - if self._is_git_repository(): - table.add(COMMIT_SHA_COLUMN) - - if self.scan_type == SECRET_SCAN_TYPE: - table.add(SECRET_SHA_COLUMN) - table.add(VIOLATION_LENGTH_COLUMN) - table.add(VIOLATION_COLUMN) - - return table - - def _enrich_table_with_values(self, table: Table, detection: Detection, document: Document) -> None: - self._enrich_table_with_detection_summary_values(table, detection, document) - self._enrich_table_with_detection_code_segment_values(table, detection, document) - - def _enrich_table_with_detection_summary_values( - self, table: Table, detection: Detection, document: Document - ) -> None: - issue_type = detection.message - if self.scan_type == SECRET_SCAN_TYPE: - issue_type = detection.type - - table.set(ISSUE_TYPE_COLUMN, issue_type) - table.set(RULE_ID_COLUMN, detection.detection_rule_id) - table.set(FILE_PATH_COLUMN, click.format_filename(document.path)) - table.set(SECRET_SHA_COLUMN, detection.detection_details.get('sha512', '')) - table.set(COMMIT_SHA_COLUMN, detection.detection_details.get('commit_id', '')) - - def _enrich_table_with_detection_code_segment_values( - self, table: Table, detection: Detection, document: Document - ) -> None: - detection_details = detection.detection_details - - detection_line = detection_details.get('line_in_file', -1) - if self.scan_type == SECRET_SCAN_TYPE: - detection_line = detection_details.get('line', -1) - - detection_column = get_position_in_line(document.content, detection_details.get('start_position', -1)) - violation_length = detection_details.get('length', -1) - - violation = '' - file_content_lines = document.content.splitlines() - if detection_line < len(file_content_lines): - line = file_content_lines[detection_line] - violation = line[detection_column: detection_column + violation_length] - - if not self.show_secret: - violation = obfuscate_text(violation) - - table.set(LINE_NUMBER_COLUMN, str(detection_line)) - table.set(COLUMN_NUMBER_COLUMN, str(detection_column)) - table.set(VIOLATION_LENGTH_COLUMN, f'{violation_length} chars') - table.set(VIOLATION_COLUMN, violation) diff --git a/cycode/cli/printers/tables/__init__.py b/cycode/cli/printers/tables/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cycode/cli/printers/tables/sca_table_printer.py b/cycode/cli/printers/tables/sca_table_printer.py new file mode 100644 index 00000000..064d21d1 --- /dev/null +++ b/cycode/cli/printers/tables/sca_table_printer.py @@ -0,0 +1,138 @@ +from collections import defaultdict +from typing import TYPE_CHECKING + +from cycode.cli.cli_types import SeverityOption +from cycode.cli.consts import LICENSE_COMPLIANCE_POLICY_ID, PACKAGE_VULNERABILITY_POLICY_ID +from cycode.cli.models import Detection +from cycode.cli.printers.tables.table import Table +from cycode.cli.printers.tables.table_models import ColumnInfoBuilder +from cycode.cli.printers.tables.table_printer_base import TablePrinterBase +from cycode.cli.printers.utils import is_git_diff_based_scan +from cycode.cli.printers.utils.detection_ordering.sca_ordering import sort_and_group_detections +from cycode.cli.utils.string_utils import shortcut_dependency_paths + +if TYPE_CHECKING: + from cycode.cli.models import LocalScanResult + +column_builder = ColumnInfoBuilder() + +# Building must have strict order. Represents the order of the columns in the table (from left to right) +SEVERITY_COLUMN = column_builder.build(name='Severity') +REPOSITORY_COLUMN = column_builder.build(name='Repository') +CODE_PROJECT_COLUMN = column_builder.build(name='Code Project', highlight=False) # File path to the manifest file +ECOSYSTEM_COLUMN = column_builder.build(name='Ecosystem', highlight=False) +PACKAGE_COLUMN = column_builder.build(name='Package', highlight=False) +CVE_COLUMNS = column_builder.build(name='CVE', highlight=False) +DEPENDENCY_PATHS_COLUMN = column_builder.build(name='Dependency Paths') +UPGRADE_COLUMN = column_builder.build(name='Upgrade') +LICENSE_COLUMN = column_builder.build(name='License', highlight=False) +DIRECT_DEPENDENCY_COLUMN = column_builder.build(name='Direct Dependency') +DEVELOPMENT_DEPENDENCY_COLUMN = column_builder.build(name='Development Dependency') + + +class ScaTablePrinter(TablePrinterBase): + def _print_results(self, local_scan_results: list['LocalScanResult']) -> None: + detections_per_policy_id = self._extract_detections_per_policy_id(local_scan_results) + for policy_id, detections in detections_per_policy_id.items(): + table = self._get_table(policy_id) + + resulting_detections, group_separator_indexes = sort_and_group_detections(detections) + for detection in resulting_detections: + self._enrich_table_with_values(table, detection) + + table.set_group_separator_indexes(group_separator_indexes) + + self._print_summary_issues(len(detections), self._get_title(policy_id)) + self._print_table(table) + + @staticmethod + def _get_title(policy_id: str) -> str: + if policy_id == PACKAGE_VULNERABILITY_POLICY_ID: + return 'Dependency Vulnerabilities' + if policy_id == LICENSE_COMPLIANCE_POLICY_ID: + return 'License Compliance' + + return 'Unknown' + + def _get_table(self, policy_id: str) -> Table: + table = Table() + + if policy_id == PACKAGE_VULNERABILITY_POLICY_ID: + table.add_column(CVE_COLUMNS) + table.add_column(UPGRADE_COLUMN) + elif policy_id == LICENSE_COMPLIANCE_POLICY_ID: + table.add_column(LICENSE_COLUMN) + + if is_git_diff_based_scan(self.command_scan_type): + table.add_column(REPOSITORY_COLUMN) + + table.add_column(SEVERITY_COLUMN) + table.add_column(CODE_PROJECT_COLUMN) + table.add_column(ECOSYSTEM_COLUMN) + table.add_column(PACKAGE_COLUMN) + table.add_column(DIRECT_DEPENDENCY_COLUMN) + table.add_column(DEVELOPMENT_DEPENDENCY_COLUMN) + table.add_column(DEPENDENCY_PATHS_COLUMN) + + return table + + @staticmethod + def _enrich_table_with_values(table: Table, detection: Detection) -> None: + detection_details = detection.detection_details + + if detection.severity: + table.add_cell(SEVERITY_COLUMN, SeverityOption(detection.severity)) + else: + table.add_cell(SEVERITY_COLUMN, 'N/A') + + table.add_cell(REPOSITORY_COLUMN, detection_details.get('repository_name')) + table.add_file_path_cell(CODE_PROJECT_COLUMN, detection_details.get('file_path')) + table.add_cell(ECOSYSTEM_COLUMN, detection_details.get('ecosystem')) + table.add_cell(PACKAGE_COLUMN, detection_details.get('package_name')) + + dependency_bool_to_color = { + True: 'green', + False: 'red', + } # by default, not colored (None) + table.add_cell( + column=DIRECT_DEPENDENCY_COLUMN, + value=detection_details.get('is_direct_dependency_str'), + color=dependency_bool_to_color.get(detection_details.get('is_direct_dependency')), + ) + table.add_cell( + column=DEVELOPMENT_DEPENDENCY_COLUMN, + value=detection_details.get('is_dev_dependency_str'), + color=dependency_bool_to_color.get(detection_details.get('is_dev_dependency')), + ) + + dependency_paths = 'N/A' + dependency_paths_raw = detection_details.get('dependency_paths') + if dependency_paths_raw: + dependency_paths = shortcut_dependency_paths(dependency_paths_raw) + table.add_cell(DEPENDENCY_PATHS_COLUMN, dependency_paths) + + upgrade = '' + alert = detection_details.get('alert') + if alert and alert.get('first_patched_version'): + upgrade = f'{alert.get("vulnerable_requirements")} -> {alert.get("first_patched_version")}' + table.add_cell(UPGRADE_COLUMN, upgrade) + + table.add_cell(CVE_COLUMNS, detection_details.get('vulnerability_id')) + table.add_cell(LICENSE_COLUMN, detection_details.get('license')) + + def _print_summary_issues(self, detections_count: int, title: str) -> None: + self.console.print(f'[bold]Cycode found {detections_count} violations of type: [cyan]{title}[/]') + + @staticmethod + def _extract_detections_per_policy_id( + local_scan_results: list['LocalScanResult'], + ) -> dict[str, list[Detection]]: + detections_to_policy_id = defaultdict(list) + + for local_scan_result in local_scan_results: + for document_detection in local_scan_result.document_detections: + for detection in document_detection.detections: + detections_to_policy_id[detection.detection_type_id].append(detection) + + # sort dict by keys (policy id) to make persist output order + return dict(sorted(detections_to_policy_id.items(), reverse=True)) diff --git a/cycode/cli/printers/tables/table.py b/cycode/cli/printers/tables/table.py new file mode 100644 index 00000000..61e143ca --- /dev/null +++ b/cycode/cli/printers/tables/table.py @@ -0,0 +1,64 @@ +import urllib.parse +from typing import TYPE_CHECKING, Optional + +from rich.markup import escape +from rich.table import Table as RichTable + +if TYPE_CHECKING: + from cycode.cli.printers.tables.table_models import ColumnInfo + + +class Table: + """Helper class to manage columns and their values in the right order and only if the column should be presented.""" + + def __init__(self, column_infos: Optional[list['ColumnInfo']] = None) -> None: + self._group_separator_indexes: set[int] = set() + + self._columns: dict[ColumnInfo, list[str]] = {} + if column_infos: + self._columns = {columns: [] for columns in column_infos} + + def add_column(self, column: 'ColumnInfo') -> None: + self._columns[column] = [] + + def _add_cell_no_error(self, column: 'ColumnInfo', value: str) -> None: + # we push values only for existing columns what were added before + if column in self._columns: + self._columns[column].append(value) + + def add_cell(self, column: 'ColumnInfo', value: str, color: Optional[str] = None) -> None: + if color: + value = f'[{color}]{value}[/]' + + self._add_cell_no_error(column, value) + + def add_file_path_cell(self, column: 'ColumnInfo', path: str) -> None: + encoded_path = urllib.parse.quote(path) + escaped_path = escape(encoded_path) + self._add_cell_no_error(column, f'[link file://{escaped_path}]{path}') + + def set_group_separator_indexes(self, group_separator_indexes: set[int]) -> None: + self._group_separator_indexes = group_separator_indexes + + def _get_ordered_columns(self) -> list['ColumnInfo']: + # we are sorting columns by index to make sure that columns will be printed in the right order + return sorted(self._columns, key=lambda column_info: column_info.index) + + def get_columns_info(self) -> list['ColumnInfo']: + return self._get_ordered_columns() + + def get_rows(self) -> list[str]: + column_values = [self._columns[column_info] for column_info in self._get_ordered_columns()] + return list(zip(*column_values)) + + def get_table(self) -> 'RichTable': + table = RichTable(expand=True, highlight=True) + + for column in self.get_columns_info(): + extra_args = column.column_opts if column.column_opts else {} + table.add_column(header=column.name, overflow='fold', **extra_args) + + for index, raw in enumerate(self.get_rows()): + table.add_row(*raw, end_section=index in self._group_separator_indexes) + + return table diff --git a/cycode/cli/printers/tables/table_models.py b/cycode/cli/printers/tables/table_models.py new file mode 100644 index 00000000..58e41aaa --- /dev/null +++ b/cycode/cli/printers/tables/table_models.py @@ -0,0 +1,25 @@ +from typing import NamedTuple, Optional + + +class ColumnInfoBuilder: + def __init__(self) -> None: + self._index = 0 + + def build(self, name: str, **column_opts) -> 'ColumnInfo': + column_info = ColumnInfo(name, self._index, column_opts) + self._index += 1 + return column_info + + +class ColumnInfo(NamedTuple): + name: str + index: int # Represents the order of the columns, starting from the left + column_opts: Optional[dict] = None + + def __hash__(self) -> int: + return hash((self.name, self.index)) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, ColumnInfo): + return NotImplemented + return (self.name, self.index) == (other.name, other.index) diff --git a/cycode/cli/printers/tables/table_printer.py b/cycode/cli/printers/tables/table_printer.py new file mode 100644 index 00000000..4468ef9f --- /dev/null +++ b/cycode/cli/printers/tables/table_printer.py @@ -0,0 +1,104 @@ +from typing import TYPE_CHECKING + +from cycode.cli.cli_types import SeverityOption +from cycode.cli.consts import SECRET_SCAN_TYPE +from cycode.cli.models import Detection, Document +from cycode.cli.printers.tables.table import Table +from cycode.cli.printers.tables.table_models import ColumnInfoBuilder +from cycode.cli.printers.tables.table_printer_base import TablePrinterBase +from cycode.cli.printers.utils import is_git_diff_based_scan +from cycode.cli.printers.utils.detection_ordering.common_ordering import sort_and_group_detections_from_scan_result +from cycode.cli.utils.string_utils import get_position_in_line, obfuscate_text, sanitize_text_for_encoding + +if TYPE_CHECKING: + from cycode.cli.models import LocalScanResult + +column_builder = ColumnInfoBuilder() + +# Building must have strict order. Represents the order of the columns in the table (from left to right) +SEVERITY_COLUMN = column_builder.build(name='Severity') +ISSUE_TYPE_COLUMN = column_builder.build(name='Issue Type') +FILE_PATH_COLUMN = column_builder.build(name='File Path', highlight=False) +LINE_NUMBER_COLUMN = column_builder.build(name='Line') +COLUMN_NUMBER_COLUMN = column_builder.build(name='Column') +VIOLATION_COLUMN = column_builder.build(name='Violation', highlight=False) +VIOLATION_LENGTH_COLUMN = column_builder.build(name='Length') +SECRET_SHA_COLUMN = column_builder.build(name='Secret SHA') +COMMIT_SHA_COLUMN = column_builder.build(name='Commit SHA') + + +class TablePrinter(TablePrinterBase): + def _print_results(self, local_scan_results: list['LocalScanResult']) -> None: + table = self._get_table() + + detections, group_separator_indexes = sort_and_group_detections_from_scan_result(local_scan_results) + for detection, document in detections: + self._enrich_table_with_values(table, detection, document) + + table.set_group_separator_indexes(group_separator_indexes) + + self._print_table(table) + + def _get_table(self) -> Table: + table = Table() + + table.add_column(SEVERITY_COLUMN) + table.add_column(ISSUE_TYPE_COLUMN) + table.add_column(FILE_PATH_COLUMN) + table.add_column(LINE_NUMBER_COLUMN) + table.add_column(COLUMN_NUMBER_COLUMN) + + if is_git_diff_based_scan(self.command_scan_type): + table.add_column(COMMIT_SHA_COLUMN) + + if self.scan_type == SECRET_SCAN_TYPE: + table.add_column(SECRET_SHA_COLUMN) + table.add_column(VIOLATION_LENGTH_COLUMN) + table.add_column(VIOLATION_COLUMN) + + return table + + def _enrich_table_with_values(self, table: Table, detection: Detection, document: Document) -> None: + self._enrich_table_with_detection_summary_values(table, detection, document) + self._enrich_table_with_detection_code_segment_values(table, detection, document) + + def _enrich_table_with_detection_summary_values( + self, table: Table, detection: Detection, document: Document + ) -> None: + issue_type = detection.message + if self.scan_type == SECRET_SCAN_TYPE: + issue_type = detection.type + + table.add_cell(SEVERITY_COLUMN, SeverityOption(detection.severity)) + table.add_cell(ISSUE_TYPE_COLUMN, issue_type) + table.add_file_path_cell(FILE_PATH_COLUMN, document.path) + table.add_cell(SECRET_SHA_COLUMN, detection.detection_details.get('sha512', '')) + table.add_cell(COMMIT_SHA_COLUMN, detection.detection_details.get('commit_id', '')) + + def _enrich_table_with_detection_code_segment_values( + self, table: Table, detection: Detection, document: Document + ) -> None: + detection_details = detection.detection_details + + detection_line = detection_details.get('line_in_file', -1) + if self.scan_type == SECRET_SCAN_TYPE: + detection_line = detection_details.get('line', -1) + + detection_column = get_position_in_line(document.content, detection_details.get('start_position', -1)) + violation_length = detection_details.get('length', -1) + + violation = '' + file_content_lines = document.content.splitlines() + if detection_line < len(file_content_lines): + line = file_content_lines[detection_line] + violation = line[detection_column : detection_column + violation_length] + + if not self.show_secret: + violation = obfuscate_text(violation) + + violation = sanitize_text_for_encoding(violation) + + table.add_cell(LINE_NUMBER_COLUMN, str(detection_line)) + table.add_cell(COLUMN_NUMBER_COLUMN, str(detection_column)) + table.add_cell(VIOLATION_LENGTH_COLUMN, f'{violation_length} chars') + table.add_cell(VIOLATION_COLUMN, violation) diff --git a/cycode/cli/printers/tables/table_printer_base.py b/cycode/cli/printers/tables/table_printer_base.py new file mode 100644 index 00000000..8cb4cbda --- /dev/null +++ b/cycode/cli/printers/tables/table_printer_base.py @@ -0,0 +1,42 @@ +import abc +from typing import TYPE_CHECKING, Optional + +from cycode.cli.models import CliError, CliResult +from cycode.cli.printers.printer_base import PrinterBase +from cycode.cli.printers.text_printer import TextPrinter + +if TYPE_CHECKING: + from cycode.cli.models import LocalScanResult + from cycode.cli.printers.tables.table import Table + + +class TablePrinterBase(PrinterBase, abc.ABC): + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + self.text_printer = TextPrinter(self.ctx, self.console, self.console_err) + + def print_result(self, result: CliResult) -> None: + self.text_printer.print_result(result) + + def print_error(self, error: CliError) -> None: + self.text_printer.print_error(error) + + def print_scan_results( + self, local_scan_results: list['LocalScanResult'], errors: Optional[dict[str, 'CliError']] = None + ) -> None: + if not errors and all(result.issue_detected == 0 for result in local_scan_results): + self.console.print(self.NO_DETECTIONS_MESSAGE) + return + + self._print_results(local_scan_results) + + self.print_scan_results_summary(local_scan_results) + self.text_printer.print_report_urls_and_errors(local_scan_results, errors) + + @abc.abstractmethod + def _print_results(self, local_scan_results: list['LocalScanResult']) -> None: + raise NotImplementedError + + def _print_table(self, table: 'Table') -> None: + if table.get_rows(): + self.console.print(table.get_table()) diff --git a/cycode/cli/printers/text_printer.py b/cycode/cli/printers/text_printer.py index c3a655d6..51da53c5 100644 --- a/cycode/cli/printers/text_printer.py +++ b/cycode/cli/printers/text_printer.py @@ -1,186 +1,137 @@ -import math -from typing import List, Optional +from typing import TYPE_CHECKING, Optional -import click +from cycode.cli import consts +from cycode.cli.cli_types import SeverityOption +from cycode.cli.models import CliError, CliResult, Document +from cycode.cli.printers.printer_base import PrinterBase +from cycode.cli.printers.utils.code_snippet_syntax import get_code_snippet_syntax, get_detection_line +from cycode.cli.printers.utils.detection_data import get_detection_title +from cycode.cli.printers.utils.detection_ordering.common_ordering import sort_and_group_detections_from_scan_result -from cycode.cli.printers.base_printer import BasePrinter -from cycode.cli.models import DocumentDetections, Detection, Document, CliResult, CliError -from cycode.cli.config import config -from cycode.cli.consts import SECRET_SCAN_TYPE, COMMIT_RANGE_BASED_COMMAND_SCAN_TYPES -from cycode.cli.utils.string_utils import obfuscate_text, get_position_in_line +if TYPE_CHECKING: + from cycode.cli.models import Detection, LocalScanResult -class TextPrinter(BasePrinter): - def __init__(self, context: click.Context): - super().__init__(context) - self.scan_id: str = context.obj.get('scan_id') - self.scan_type: str = context.obj.get('scan_type') - self.command_scan_type: str = context.info_name - self.show_secret: bool = context.obj.get('show_secret', False) - +class TextPrinter(PrinterBase): def print_result(self, result: CliResult) -> None: - color = None + color = 'default' if not result.success: - color = self.RED_COLOR_NAME + color = 'red' - click.secho(result.message, fg=color) + self.console.print(result.message, style=color) - def print_error(self, error: CliError) -> None: - click.secho(error.message, fg=self.RED_COLOR_NAME, nl=False) + if not result.data: + return - def print_scan_results(self, results: List[DocumentDetections]): - click.secho(f"Scan Results: (scan_id: {self.scan_id})") + self.console.print('\nAdditional data:', style=color) + for name, value in result.data.items(): + self.console.print(f'- {name}: {value}', style=color) - if not results: - click.secho("Good job! No issues were found!!! 👏👏👏", fg=self.GREEN_COLOR_NAME) + def print_error(self, error: CliError) -> None: + self.console.print(f'[red]Error: {error.message}[/]', highlight=False) + + def print_scan_results( + self, local_scan_results: list['LocalScanResult'], errors: Optional[dict[str, 'CliError']] = None + ) -> None: + if not errors and all(result.issue_detected == 0 for result in local_scan_results): + self.console.print(self.NO_DETECTIONS_MESSAGE) return - for document_detections in results: - self._print_document_detections(document_detections) + detections, _ = sort_and_group_detections_from_scan_result(local_scan_results) + for detection, document in detections: + self.__print_document_detection(document, detection) + + self.print_scan_results_summary(local_scan_results) + self.print_report_urls_and_errors(local_scan_results, errors) - report_url = self.context.obj.get('report_url') - if report_url: - click.secho(f'Report URL: {report_url}') + def __print_document_detection(self, document: 'Document', detection: 'Detection') -> None: + self.__print_detection_summary(detection, document.path) + self.__print_detection_code_segment(detection, document) + self._print_new_line() - def _print_document_detections(self, document_detections: DocumentDetections): - document = document_detections.document - lines_to_display = self._get_lines_to_display_count() - for detection in document_detections.detections: - self._print_detection_summary(detection, document.path) - self._print_detection_code_segment(detection, document, lines_to_display) + def _print_new_line(self) -> None: + self.console.line() + + def __print_detection_summary(self, detection: 'Detection', document_path: str) -> None: + title = get_detection_title(self.scan_type, detection) + + severity = SeverityOption(detection.severity) if detection.severity else 'N/A' + severity_icon = SeverityOption.get_member_emoji(detection.severity) if detection.severity else '' + + line_no = get_detection_line(self.scan_type, detection) + 1 + clickable_document_path = f'[u]{document_path}:{line_no}[/]' - def _print_detection_summary(self, detection: Detection, document_path: str): - detection_name = detection.type if self.scan_type == SECRET_SCAN_TYPE else detection.message - detection_sha = detection.detection_details.get('sha512') - detection_sha_message = f'\nSecret SHA: {detection_sha}' if detection_sha else '' detection_commit_id = detection.detection_details.get('commit_id') detection_commit_id_message = f'\nCommit SHA: {detection_commit_id}' if detection_commit_id else '' - click.echo( - f'⛔ Found issue of type: {click.style(detection_name, fg="bright_red", bold=True)} ' - f'(rule ID: {detection.detection_rule_id}) in file: {click.format_filename(document_path)} ' - f'{detection_sha_message}{detection_commit_id_message} ⛔' + + self.console.print( + severity_icon, + severity, + f'violation: [b bright_red]{title}[/]{detection_commit_id_message}\n', + *self.__get_intermediate_summary_lines(detection), + f'[dodger_blue1]File: {clickable_document_path}[/]', ) - def _print_detection_code_segment(self, detection: Detection, document: Document, code_segment_size: int): - if self._is_git_diff_based_scan(): - self._print_detection_from_git_diff(detection, document) - return + def __get_intermediate_summary_lines(self, detection: 'Detection') -> list[str]: + intermediate_summary_lines = [] - self._print_detection_from_file(detection, document, code_segment_size) + if self.scan_type == consts.SCA_SCAN_TYPE: + intermediate_summary_lines.extend(self.__get_sca_related_summary_lines(detection)) - def _get_code_segment_start_line(self, detection_line: int, code_segment_size: int): - start_line = detection_line - math.ceil(code_segment_size / 2) - return 0 if start_line < 0 else start_line + return intermediate_summary_lines - def _print_line_of_code_segment(self, document: Document, line: str, line_number: int, - detection_position_in_line: int, violation_length: int, is_detection_line: bool): - if is_detection_line: - self._print_detection_line(document, line, line_number, detection_position_in_line, violation_length) - else: - self._print_line(document, line, line_number) + @staticmethod + def __get_sca_related_summary_lines(detection: 'Detection') -> list[str]: + summary_lines = [] + + if detection.has_alert: + patched_version = detection.detection_details['alert'].get('first_patched_version') + patched_version = patched_version or 'Not fixed' - def _print_detection_line(self, document: Document, line: str, line_number: int, detection_position_in_line: int, - violation_length: int) -> None: - click.echo( - f'{self._get_line_number_style(line_number)} ' - f'{self._get_detection_line_style(line, document.is_git_diff_format, detection_position_in_line, violation_length)}' + summary_lines.append(f'First patched version: [cyan]{patched_version}[/]\n') + else: + package_license = detection.detection_details.get('license', 'N/A') + summary_lines.append(f'License: [cyan]{package_license}[/]\n') + + return summary_lines + + def __print_detection_code_segment(self, detection: 'Detection', document: Document) -> None: + self.console.print( + get_code_snippet_syntax( + self.scan_type, + self.command_scan_type, + detection, + document, + obfuscate=not self.show_secret, + ) ) - def _print_line(self, document: Document, line: str, line_number: int): - line_no = self._get_line_number_style(line_number) - line = self._get_line_style(line, document.is_git_diff_format) - - click.echo(f'{line_no} {line}') - - def _get_detection_line_style(self, line: str, is_git_diff: bool, start_position: int, length: int) -> str: - line_color = self._get_line_color(line, is_git_diff) - if self.scan_type != SECRET_SCAN_TYPE or start_position < 0 or length < 0: - return self._get_line_style(line, is_git_diff, line_color) - - violation = line[start_position: start_position + length] - if not self.show_secret: - violation = obfuscate_text(violation) - - line_to_violation = line[0: start_position] - line_from_violation = line[start_position + length:] - - return f'{self._get_line_style(line_to_violation, is_git_diff, line_color)}' \ - f'{self._get_line_style(violation, is_git_diff, line_color, underline=True)}' \ - f'{self._get_line_style(line_from_violation, is_git_diff, line_color)}' - - def _get_line_style( - self, line: str, is_git_diff: bool, color: Optional[str] = None, underline: bool = False - ) -> str: - if color is None: - color = self._get_line_color(line, is_git_diff) - - return click.style(line, fg=color, bold=False, underline=underline) - - def _get_line_color(self, line: str, is_git_diff: bool) -> str: - if not is_git_diff: - return self.WHITE_COLOR_NAME - - if line.startswith('+'): - return self.GREEN_COLOR_NAME - - if line.startswith('-'): - return self.RED_COLOR_NAME - - return self.WHITE_COLOR_NAME - - def _get_line_number_style(self, line_number: int): - return f'{click.style(str(line_number), fg=self.WHITE_COLOR_NAME, bold=False)} ' \ - f'{click.style("|", fg=self.RED_COLOR_NAME, bold=False)}' - - def _get_lines_to_display_count(self) -> int: - result_printer_configuration = config.get('result_printer') - lines_to_display_of_scan = result_printer_configuration.get(self.scan_type, {}) \ - .get(self.command_scan_type, {}).get('lines_to_display') - if lines_to_display_of_scan: - return lines_to_display_of_scan - - return result_printer_configuration.get('default').get('lines_to_display') - - def _print_detection_from_file(self, detection: Detection, document: Document, code_segment_size: int): - detection_details = detection.detection_details - detection_line = detection_details.get('line', -1) if self.scan_type == SECRET_SCAN_TYPE else \ - detection_details.get('line_in_file', -1) - detection_position = detection_details.get('start_position', -1) - violation_length = detection_details.get('length', -1) - - file_content = document.content - file_lines = file_content.splitlines() - start_line = self._get_code_segment_start_line(detection_line, code_segment_size) - detection_position_in_line = get_position_in_line(file_content, detection_position) - - click.echo() - for i in range(code_segment_size): - current_line_index = start_line + i - if current_line_index >= len(file_lines): - break - - current_line = file_lines[current_line_index] - is_detection_line = current_line_index == detection_line - self._print_line_of_code_segment(document, current_line, current_line_index + 1, detection_position_in_line, - violation_length, is_detection_line) - click.echo() - - def _print_detection_from_git_diff(self, detection: Detection, document: Document): - detection_details = detection.detection_details - detection_line_number = detection_details.get('line', -1) - detection_line_number_in_original_file = detection_details.get('line_in_file', -1) - detection_position = detection_details.get('start_position', -1) - violation_length = detection_details.get('length', -1) - - git_diff_content = document.content - git_diff_lines = git_diff_content.splitlines() - detection_line = git_diff_lines[detection_line_number] - detection_position_in_line = get_position_in_line(git_diff_content, detection_position) - - click.echo() - self._print_detection_line(document, detection_line, detection_line_number_in_original_file, - detection_position_in_line, violation_length) - click.echo() - - def _is_git_diff_based_scan(self): - return self.command_scan_type in COMMIT_RANGE_BASED_COMMAND_SCAN_TYPES and self.scan_type == SECRET_SCAN_TYPE + def print_report_urls_and_errors( + self, local_scan_results: list['LocalScanResult'], errors: Optional[dict[str, 'CliError']] = None + ) -> None: + report_urls = [scan_result.report_url for scan_result in local_scan_results if scan_result.report_url] + + self.print_report_urls(report_urls, self.ctx.obj.get('aggregation_report_url')) + if not errors: + return + + self.console.print(self.FAILED_SCAN_MESSAGE) + for scan_id, error in errors.items(): + self.console.print(f'- {scan_id}: ', end='') + self.print_error(error) + + def print_report_urls(self, report_urls: list[str], aggregation_report_url: Optional[str] = None) -> None: + if not report_urls and not aggregation_report_url: + return + + # Prioritize aggregation report URL; if report urls is only one, use it instead + single_url = report_urls[0] if len(report_urls) == 1 else None + single_url = aggregation_report_url or single_url + if single_url: + self.console.print(f'[b]Report URL:[/] {single_url}') + return + + # If there are multiple report URLs, print them all + self.console.print('[b]Report URLs:[/]') + for report_url in report_urls: + self.console.print(f'- {report_url}') diff --git a/cycode/cli/printers/utils/__init__.py b/cycode/cli/printers/utils/__init__.py new file mode 100644 index 00000000..d1ee86c0 --- /dev/null +++ b/cycode/cli/printers/utils/__init__.py @@ -0,0 +1,5 @@ +from cycode.cli import consts + + +def is_git_diff_based_scan(command_scan_type: str) -> bool: + return command_scan_type in consts.COMMIT_RANGE_BASED_COMMAND_SCAN_TYPES diff --git a/cycode/cli/printers/utils/code_snippet_syntax.py b/cycode/cli/printers/utils/code_snippet_syntax.py new file mode 100644 index 00000000..57bc084e --- /dev/null +++ b/cycode/cli/printers/utils/code_snippet_syntax.py @@ -0,0 +1,122 @@ +from typing import TYPE_CHECKING + +from rich.syntax import Syntax + +from cycode.cli import consts +from cycode.cli.console import _SYNTAX_HIGHLIGHT_THEME +from cycode.cli.printers.utils import is_git_diff_based_scan +from cycode.cli.utils.string_utils import get_position_in_line, obfuscate_text, sanitize_text_for_encoding + +if TYPE_CHECKING: + from cycode.cli.models import Document + from cycode.cyclient.models import Detection + + +def _get_code_segment_start_line(detection_line: int, lines_to_display_before: int) -> int: + start_line = detection_line - lines_to_display_before + return 0 if start_line < 0 else start_line + + +def get_detection_line(scan_type: str, detection: 'Detection') -> int: + return ( + detection.detection_details.get('line', -1) + if scan_type == consts.SECRET_SCAN_TYPE + else detection.detection_details.get('line_in_file', -1) - 1 + ) + + +def _get_syntax_highlighted_code(code: str, lexer: str, start_line: int, detection_line: int) -> Syntax: + return Syntax( + theme=_SYNTAX_HIGHLIGHT_THEME, + code=code, + lexer=lexer, + line_numbers=True, + word_wrap=True, + dedent=True, + tab_size=2, + start_line=start_line + 1, + highlight_lines={detection_line + 1}, + ) + + +def _get_code_snippet_syntax_from_file( + scan_type: str, + detection: 'Detection', + document: 'Document', + lines_to_display_before: int, + lines_to_display_after: int, + obfuscate: bool, +) -> Syntax: + detection_details = detection.detection_details + detection_line = get_detection_line(scan_type, detection) + start_line_index = _get_code_segment_start_line(detection_line, lines_to_display_before) + detection_position = get_position_in_line(document.content, detection_details.get('start_position', -1)) + violation_length = detection_details.get('length', -1) + + code_lines_to_render = [] + document_content_lines = document.content.splitlines() + total_lines_to_display = lines_to_display_before + 1 + lines_to_display_after + + for line_index in range(total_lines_to_display): + current_line_index = start_line_index + line_index + if current_line_index >= len(document_content_lines): + break + + line_content = document_content_lines[current_line_index] + + line_with_detection = current_line_index == detection_line + if scan_type == consts.SECRET_SCAN_TYPE and line_with_detection and obfuscate: + violation = line_content[detection_position : detection_position + violation_length] + code_lines_to_render.append(line_content.replace(violation, obfuscate_text(violation))) + else: + code_lines_to_render.append(line_content) + + code_to_render = '\n'.join(code_lines_to_render) + code_to_render = sanitize_text_for_encoding(code_to_render) + return _get_syntax_highlighted_code( + code=code_to_render, + lexer=Syntax.guess_lexer(document.path, code=code_to_render), + start_line=start_line_index, + detection_line=detection_line, + ) + + +def _get_code_snippet_syntax_from_git_diff( + scan_type: str, detection: 'Detection', document: 'Document', obfuscate: bool +) -> Syntax: + detection_details = detection.detection_details + detection_line = get_detection_line(scan_type, detection) + detection_position = detection_details.get('start_position', -1) + violation_length = detection_details.get('length', -1) + + line_content = document.content.splitlines()[detection_line] + detection_position_in_line = get_position_in_line(document.content, detection_position) + if scan_type == consts.SECRET_SCAN_TYPE and obfuscate: + violation = line_content[detection_position_in_line : detection_position_in_line + violation_length] + line_content = line_content.replace(violation, obfuscate_text(violation)) + + line_content = sanitize_text_for_encoding(line_content) + return _get_syntax_highlighted_code( + code=line_content, + lexer='diff', + start_line=detection_line, + detection_line=detection_line, + ) + + +def get_code_snippet_syntax( + scan_type: str, + command_scan_type: str, + detection: 'Detection', + document: 'Document', + lines_to_display_before: int = 1, + lines_to_display_after: int = 1, + obfuscate: bool = True, +) -> Syntax: + if is_git_diff_based_scan(command_scan_type): + # it will return syntax with just one line + return _get_code_snippet_syntax_from_git_diff(scan_type, detection, document, obfuscate) + + return _get_code_snippet_syntax_from_file( + scan_type, detection, document, lines_to_display_before, lines_to_display_after, obfuscate + ) diff --git a/cycode/cli/printers/utils/detection_data.py b/cycode/cli/printers/utils/detection_data.py new file mode 100644 index 00000000..679429a3 --- /dev/null +++ b/cycode/cli/printers/utils/detection_data.py @@ -0,0 +1,108 @@ +from pathlib import Path +from typing import TYPE_CHECKING, Optional + +from cycode.cli import consts + +if TYPE_CHECKING: + from cycode.cyclient.models import Detection + + +def get_cwe_cve_link(cwe_cve: Optional[str]) -> Optional[str]: + if not cwe_cve: + return None + + if cwe_cve.startswith('GHSA'): + return f'https://github.com/advisories/{cwe_cve}' + + if cwe_cve.startswith('CWE'): + # string example: 'CWE-532: Insertion of Sensitive Information into Log File' + parts = cwe_cve.split('-') + if len(parts) < 1: + return None + + number = '' + for char in parts[1]: + if char.isdigit(): + number += char + else: + break + + return f'https://cwe.mitre.org/data/definitions/{number}' + + if cwe_cve.startswith('CVE'): + return f'https://cve.mitre.org/cgi-bin/cvename.cgi?name={cwe_cve}' + + return None + + +def clear_cwe_name(cwe: str) -> str: + """Clear CWE. + + Intput: CWE-532: Insertion of Sensitive Information into Log File + Output: CWE-532 + """ + if cwe.startswith('CWE'): + return cwe.split(':')[0] + + return cwe + + +def get_detection_clickable_cwe_cve(scan_type: str, detection: 'Detection') -> str: + def link(url: str, name: str) -> str: + return f'[link={url}]{clear_cwe_name(name)}[/]' + + if scan_type == consts.SCA_SCAN_TYPE: + cve = detection.detection_details.get('vulnerability_id') + return link(get_cwe_cve_link(cve), cve) if cve else '' + if scan_type == consts.SAST_SCAN_TYPE: + renderables = [] + for cwe in detection.detection_details.get('cwe', []): + cwe and renderables.append(link(get_cwe_cve_link(cwe), cwe)) + return ', '.join(renderables) + + return '' + + +def get_detection_cwe_cve(scan_type: str, detection: 'Detection') -> Optional[str]: + if scan_type == consts.SCA_SCAN_TYPE: + return detection.detection_details.get('vulnerability_id') + if scan_type == consts.SAST_SCAN_TYPE: + cwes = detection.detection_details.get('cwe') # actually it is List[str] + if not cwes: + return None + + return ' | '.join(cwes) + + return None + + +def get_detection_title(scan_type: str, detection: 'Detection') -> str: + title = detection.message + if scan_type == consts.SAST_SCAN_TYPE: + title = detection.detection_details['policy_display_name'] + elif scan_type == consts.SECRET_SCAN_TYPE: + title = f'Hardcoded {detection.type} is used' + + is_sca_package_vulnerability = scan_type == consts.SCA_SCAN_TYPE and detection.has_alert + if is_sca_package_vulnerability: + title = detection.detection_details['alert'].get('summary', 'N/A') + + cwe_cve = get_detection_cwe_cve(scan_type, detection) + return f'[{cwe_cve}] {title}' if cwe_cve else title + + +def get_detection_file_path(scan_type: str, detection: 'Detection') -> Path: + if scan_type == consts.SECRET_SCAN_TYPE: + folder_path = detection.detection_details.get('file_path', '') + file_name = detection.detection_details.get('file_name', '') + return Path.joinpath(Path(folder_path), Path(file_name)) + if scan_type == consts.SAST_SCAN_TYPE: + file_path = detection.detection_details.get('file_path', '') + + # fix the absolute path...BE returns string which does not start with / + if not file_path.startswith('/'): + file_path = f'/{file_path}' + + return Path(file_path) + + return Path(detection.detection_details.get('file_path', '')) diff --git a/cycode/cli/printers/utils/detection_ordering/__init__.py b/cycode/cli/printers/utils/detection_ordering/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cycode/cli/printers/utils/detection_ordering/common_ordering.py b/cycode/cli/printers/utils/detection_ordering/common_ordering.py new file mode 100644 index 00000000..c4b431ef --- /dev/null +++ b/cycode/cli/printers/utils/detection_ordering/common_ordering.py @@ -0,0 +1,57 @@ +from typing import TYPE_CHECKING + +from cycode.cli.cli_types import SeverityOption + +if TYPE_CHECKING: + from cycode.cli.models import Document, LocalScanResult + from cycode.cyclient.models import Detection + + +GroupedDetections = tuple[list[tuple['Detection', 'Document']], set[int]] + + +def __severity_sort_key(detection_with_document: tuple['Detection', 'Document']) -> int: + detection, _ = detection_with_document + severity = detection.severity if detection.severity else '' + return SeverityOption.get_member_weight(severity) + + +def _sort_detections_by_severity( + detections_with_documents: list[tuple['Detection', 'Document']], +) -> list[tuple['Detection', 'Document']]: + return sorted(detections_with_documents, key=__severity_sort_key, reverse=True) + + +def __file_path_sort_key(detection_with_document: tuple['Detection', 'Document']) -> str: + _, document = detection_with_document + return document.path + + +def _sort_detections_by_file_path( + detections_with_documents: list[tuple['Detection', 'Document']], +) -> list[tuple['Detection', 'Document']]: + return sorted(detections_with_documents, key=__file_path_sort_key) + + +def sort_and_group_detections( + detections_with_documents: list[tuple['Detection', 'Document']], +) -> GroupedDetections: + """Sort detections by severity. We do not have grouping here (don't find the best one yet).""" + group_separator_indexes = set() + + # we sort detections by file path to make persist output order + sorted_by_path_detections = _sort_detections_by_file_path(detections_with_documents) + sorted_by_severity = _sort_detections_by_severity(sorted_by_path_detections) + + return sorted_by_severity, group_separator_indexes + + +def sort_and_group_detections_from_scan_result(local_scan_results: list['LocalScanResult']) -> GroupedDetections: + detections_with_documents = [] + for local_scan_result in local_scan_results: + for document_detections in local_scan_result.document_detections: + detections_with_documents.extend( + [(detection, document_detections.document) for detection in document_detections.detections] + ) + + return sort_and_group_detections(detections_with_documents) diff --git a/cycode/cli/printers/utils/detection_ordering/sca_ordering.py b/cycode/cli/printers/utils/detection_ordering/sca_ordering.py new file mode 100644 index 00000000..9e1f8022 --- /dev/null +++ b/cycode/cli/printers/utils/detection_ordering/sca_ordering.py @@ -0,0 +1,59 @@ +from collections import defaultdict +from typing import TYPE_CHECKING + +from cycode.cli.cli_types import SeverityOption + +if TYPE_CHECKING: + from cycode.cyclient.models import Detection + + +def __group_by(detections: list['Detection'], details_field_name: str) -> dict[str, list['Detection']]: + grouped = defaultdict(list) + for detection in detections: + grouped[detection.detection_details.get(details_field_name)].append(detection) + return grouped + + +def __severity_sort_key(detection: 'Detection') -> int: + severity = detection.severity if detection.severity else 'unknown' + return SeverityOption.get_member_weight(severity) + + +def _sort_detections_by_severity(detections: list['Detection']) -> list['Detection']: + return sorted(detections, key=__severity_sort_key, reverse=True) + + +def __package_sort_key(detection: 'Detection') -> int: + return detection.detection_details.get('package_name') + + +def _sort_detections_by_package(detections: list['Detection']) -> list['Detection']: + return sorted(detections, key=__package_sort_key) + + +def sort_and_group_detections(detections: list['Detection']) -> tuple[list['Detection'], set[int]]: + """Sort detections by severity and group by repository, code project and package name. + + Note: + Code Project is path to the manifest file. + + Grouping by code projects also groups by ecosystem. + Because manifest files are unique per ecosystem. + + """ + resulting_detections = [] + group_separator_indexes = set() + + # we sort detections by package name to make persist output order + sorted_detections = _sort_detections_by_package(detections) + + grouped_by_repository = __group_by(sorted_detections, 'repository_name') + for repository_group in grouped_by_repository.values(): + grouped_by_code_project = __group_by(repository_group, 'file_path') + for code_project_group in grouped_by_code_project.values(): + grouped_by_package = __group_by(code_project_group, 'package_name') + for package_group in grouped_by_package.values(): + group_separator_indexes.add(len(resulting_detections) - 1) # indexing starts from 0 + resulting_detections.extend(_sort_detections_by_severity(package_group)) + + return resulting_detections, group_separator_indexes diff --git a/cycode/cli/printers/utils/rich_helpers.py b/cycode/cli/printers/utils/rich_helpers.py new file mode 100644 index 00000000..6049b211 --- /dev/null +++ b/cycode/cli/printers/utils/rich_helpers.py @@ -0,0 +1,39 @@ +from typing import TYPE_CHECKING + +from rich.columns import Columns +from rich.markdown import Markdown +from rich.panel import Panel + +from cycode.cli.console import console +from cycode.cli.utils.string_utils import sanitize_text_for_encoding + +if TYPE_CHECKING: + from rich.console import RenderableType + + +def get_panel(renderable: 'RenderableType', title: str) -> Panel: + return Panel( + renderable, + title=title, + title_align='left', + border_style='dim', + ) + + +def get_markdown_panel(markdown_text: str, title: str) -> Panel: + sanitized_text = sanitize_text_for_encoding(markdown_text.strip()) + return get_panel( + Markdown(sanitized_text), + title=title, + ) + + +def get_columns_in_1_to_3_ratio(left: 'Panel', right: 'Panel', panel_border_offset: int = 5) -> Columns: + terminal_width = console.width + one_third_width = terminal_width // 3 + two_thirds_width = terminal_width - one_third_width - panel_border_offset + + left.width = one_third_width + right.width = two_thirds_width + + return Columns([left, right]) diff --git a/cycode/cli/user_settings/base_file_manager.py b/cycode/cli/user_settings/base_file_manager.py index 22c4fffc..6a9d0fe2 100644 --- a/cycode/cli/user_settings/base_file_manager.py +++ b/cycode/cli/user_settings/base_file_manager.py @@ -1,21 +1,28 @@ import os from abc import ABC, abstractmethod -from cycode.cli.utils.yaml_utils import update_file, read_file +from collections.abc import Hashable +from typing import Any +from cycode.cli.utils.yaml_utils import read_yaml_file, update_yaml_file +from cycode.logger import get_logger + +logger = get_logger('Base File Manager') -class BaseFileManager(ABC): +class BaseFileManager(ABC): @abstractmethod - def get_filename(self): - pass + def get_filename(self) -> str: ... - def read_file(self): - try: - return read_file(self.get_filename()) - except FileNotFoundError: - return {} + def read_file(self) -> dict[Hashable, Any]: + return read_yaml_file(self.get_filename()) - def write_content_to_file(self, content): + def write_content_to_file(self, content: dict[Hashable, Any]) -> None: filename = self.get_filename() - os.makedirs(os.path.dirname(filename), exist_ok=True) - update_file(filename, content) + + try: + os.makedirs(os.path.dirname(filename), exist_ok=True) + except Exception as e: + logger.warning('Failed to create directory for file, %s', {'filename': filename}, exc_info=e) + return + + update_yaml_file(filename, content) diff --git a/cycode/cli/user_settings/config_file_manager.py b/cycode/cli/user_settings/config_file_manager.py index 7c99f033..5b029e39 100644 --- a/cycode/cli/user_settings/config_file_manager.py +++ b/cycode/cli/user_settings/config_file_manager.py @@ -1,8 +1,12 @@ import os -from typing import Optional, List, Dict +from collections.abc import Hashable +from typing import TYPE_CHECKING, Any, Optional, Union -from cycode.cli.user_settings.base_file_manager import BaseFileManager from cycode.cli.consts import CYCODE_CONFIGURATION_DIRECTORY +from cycode.cli.user_settings.base_file_manager import BaseFileManager + +if TYPE_CHECKING: + from pathlib import Path class ConfigFileManager(BaseFileManager): @@ -22,66 +26,56 @@ class ConfigFileManager(BaseFileManager): COMMAND_TIMEOUT_FIELD_NAME: str = 'command_timeout' EXCLUDE_DETECTIONS_IN_DELETED_LINES: str = 'exclude_detections_in_deleted_lines' - def __init__(self, path): + def __init__(self, path: Union['Path', str]) -> None: self.path = path - def get_api_url(self) -> Optional[str]: + def get_api_url(self) -> Optional[Any]: return self._get_value_from_environment_section(self.API_URL_FIELD_NAME) - def get_app_url(self) -> Optional[str]: + def get_app_url(self) -> Optional[Any]: return self._get_value_from_environment_section(self.APP_URL_FIELD_NAME) - def get_verbose_flag(self) -> Optional[bool]: + def get_verbose_flag(self) -> Optional[Any]: return self._get_value_from_environment_section(self.VERBOSE_FIELD_NAME) - def get_exclusions_by_scan_type(self, scan_type) -> Dict: + def get_exclusions_by_scan_type(self, scan_type: str) -> dict[Hashable, Any]: exclusions_section = self._get_section(self.EXCLUSIONS_SECTION_NAME) - scan_type_exclusions = exclusions_section.get(scan_type, {}) - return scan_type_exclusions + return exclusions_section.get(scan_type, {}) - def get_max_commits(self, command_scan_type) -> Optional[int]: + def get_max_commits(self, command_scan_type: str) -> Optional[Any]: return self._get_value_from_command_scan_type_configuration(command_scan_type, self.MAX_COMMITS_FIELD_NAME) - def get_command_timeout(self, command_scan_type) -> Optional[int]: + def get_command_timeout(self, command_scan_type: str) -> Optional[Any]: return self._get_value_from_command_scan_type_configuration(command_scan_type, self.COMMAND_TIMEOUT_FIELD_NAME) - def get_exclude_detections_in_deleted_lines(self, command_scan_type) -> Optional[bool]: - return self._get_value_from_command_scan_type_configuration(command_scan_type, - self.EXCLUDE_DETECTIONS_IN_DELETED_LINES) + def get_exclude_detections_in_deleted_lines(self, command_scan_type: str) -> Optional[Any]: + return self._get_value_from_command_scan_type_configuration( + command_scan_type, self.EXCLUDE_DETECTIONS_IN_DELETED_LINES + ) + + def update_api_base_url(self, api_url: str) -> None: + update_data = {self.ENVIRONMENT_SECTION_NAME: {self.API_URL_FIELD_NAME: api_url}} + self.write_content_to_file(update_data) - def update_base_url(self, base_url: str): - update_data = { - self.ENVIRONMENT_SECTION_NAME: { - self.API_URL_FIELD_NAME: base_url - } - } + def update_app_base_url(self, app_url: str) -> None: + update_data = {self.ENVIRONMENT_SECTION_NAME: {self.APP_URL_FIELD_NAME: app_url}} self.write_content_to_file(update_data) def get_installation_id(self) -> Optional[str]: return self._get_value_from_environment_section(self.INSTALLATION_ID_FIELD_NAME) def update_installation_id(self, installation_id: str) -> None: - update_data = { - self.ENVIRONMENT_SECTION_NAME: { - self.INSTALLATION_ID_FIELD_NAME: installation_id - } - } + update_data = {self.ENVIRONMENT_SECTION_NAME: {self.INSTALLATION_ID_FIELD_NAME: installation_id}} self.write_content_to_file(update_data) - def add_exclusion(self, scan_type, exclusion_type, new_exclusion): + def add_exclusion(self, scan_type: str, exclusion_type: str, new_exclusion: str) -> None: exclusions = self._get_exclusions_by_exclusion_type(scan_type, exclusion_type) if new_exclusion in exclusions: return exclusions.append(new_exclusion) - update_data = { - self.EXCLUSIONS_SECTION_NAME: { - scan_type: { - exclusion_type: exclusions - } - } - } + update_data = {self.EXCLUSIONS_SECTION_NAME: {scan_type: {exclusion_type: exclusions}}} self.write_content_to_file(update_data) def get_config_directory_path(self) -> str: @@ -94,24 +88,22 @@ def get_filename(self) -> str: def get_config_file_route() -> str: return os.path.join(ConfigFileManager.CYCODE_HIDDEN_DIRECTORY, ConfigFileManager.FILE_NAME) - def _get_exclusions_by_exclusion_type(self, scan_type, exclusion_type) -> List: + def _get_exclusions_by_exclusion_type(self, scan_type: str, exclusion_type: str) -> list[Any]: scan_type_exclusions = self.get_exclusions_by_scan_type(scan_type) return scan_type_exclusions.get(exclusion_type, []) - def _get_value_from_environment_section(self, field_name: str): + def _get_value_from_environment_section(self, field_name: str) -> Optional[Any]: environment_section = self._get_section(self.ENVIRONMENT_SECTION_NAME) - value = environment_section.get(field_name) - return value + return environment_section.get(field_name) - def _get_scan_configuration_by_scan_type(self, command_scan_type: str) -> Dict: + def _get_scan_configuration_by_scan_type(self, command_scan_type: str) -> dict[Hashable, Any]: scan_section = self._get_section(self.SCAN_SECTION_NAME) return scan_section.get(command_scan_type, {}) - def _get_value_from_command_scan_type_configuration(self, command_scan_type: str, field_name: str): + def _get_value_from_command_scan_type_configuration(self, command_scan_type: str, field_name: str) -> Optional[Any]: command_scan_type_configuration = self._get_scan_configuration_by_scan_type(command_scan_type) - value = command_scan_type_configuration.get(field_name) - return value + return command_scan_type_configuration.get(field_name) - def _get_section(self, section_name: str) -> Dict: + def _get_section(self, section_name: str) -> dict[Hashable, Any]: file_content = self.read_file() return file_content.get(section_name, {}) diff --git a/cycode/cli/user_settings/configuration_manager.py b/cycode/cli/user_settings/configuration_manager.py index 3769a77f..689ec0d5 100644 --- a/cycode/cli/user_settings/configuration_manager.py +++ b/cycode/cli/user_settings/configuration_manager.py @@ -1,17 +1,18 @@ import os +from functools import cache from pathlib import Path -from typing import Optional, Dict +from typing import Any, Optional from uuid import uuid4 +from cycode.cli import consts from cycode.cli.user_settings.config_file_manager import ConfigFileManager -from cycode.cli.consts import * class ConfigurationManager: global_config_file_manager: ConfigFileManager local_config_file_manager: ConfigFileManager - def __init__(self): + def __init__(self) -> None: self.global_config_file_manager = ConfigFileManager(Path.home()) self.local_config_file_manager = ConfigFileManager(os.getcwd()) @@ -28,7 +29,7 @@ def get_cycode_api_url(self) -> str: if api_url is not None: return api_url - return DEFAULT_CYCODE_API_URL + return consts.DEFAULT_CYCODE_API_URL def get_cycode_app_url(self) -> str: app_url = self.get_app_url_from_environment_variables() @@ -43,7 +44,14 @@ def get_cycode_app_url(self) -> str: if app_url is not None: return app_url - return DEFAULT_CYCODE_APP_URL + return consts.DEFAULT_CYCODE_APP_URL + + def get_debug_flag_from_environment_variables(self) -> bool: + value = self._get_value_from_environment_variables(consts.DEBUG_ENV_VAR_NAME, '') + return value.lower() in {'true', '1'} + + def get_debug_flag(self) -> bool: + return self.get_debug_flag_from_environment_variables() def get_verbose_flag(self) -> bool: verbose_flag_env_var = self.get_verbose_flag_from_environment_variables() @@ -52,31 +60,29 @@ def get_verbose_flag(self) -> bool: return verbose_flag_env_var or verbose_flag_local_config or verbose_flag_global_config def get_api_url_from_environment_variables(self) -> Optional[str]: - return self._get_value_from_environment_variables(CYCODE_API_URL_ENV_VAR_NAME) + return self._get_value_from_environment_variables(consts.CYCODE_API_URL_ENV_VAR_NAME) def get_app_url_from_environment_variables(self) -> Optional[str]: - return self._get_value_from_environment_variables(CYCODE_APP_URL_ENV_VAR_NAME) + return self._get_value_from_environment_variables(consts.CYCODE_APP_URL_ENV_VAR_NAME) def get_verbose_flag_from_environment_variables(self) -> bool: - value = self._get_value_from_environment_variables(VERBOSE_ENV_VAR_NAME, '') + value = self._get_value_from_environment_variables(consts.VERBOSE_ENV_VAR_NAME, '') return value.lower() in ('true', '1') - def get_exclusions_by_scan_type(self, scan_type) -> Dict: + @cache # noqa: B019 + def get_exclusions_by_scan_type(self, scan_type: str) -> dict: local_exclusions = self.local_config_file_manager.get_exclusions_by_scan_type(scan_type) global_exclusions = self.global_config_file_manager.get_exclusions_by_scan_type(scan_type) return self._merge_exclusions(local_exclusions, global_exclusions) - def add_exclusion(self, scope: str, scan_type: str, exclusion_type: str, value: str): + def add_exclusion(self, scope: str, scan_type: str, exclusion_type: str, value: str) -> None: config_file_manager = self.get_config_file_manager(scope) config_file_manager.add_exclusion(scan_type, exclusion_type, value) - def _merge_exclusions(self, local_exclusions: Dict, global_exclusions: Dict) -> Dict: + @staticmethod + def _merge_exclusions(local_exclusions: dict, global_exclusions: dict) -> dict: keys = set(list(local_exclusions.keys()) + list(global_exclusions.keys())) - return {key: local_exclusions.get(key, []) + global_exclusions.get(key, []) for key in keys} - - def update_base_url(self, base_url: str, scope: str = 'local'): - config_file_manager = self.get_config_file_manager(scope) - config_file_manager.update_base_url(base_url) + return {key: (local_exclusions.get(key) or []) + (global_exclusions.get(key) or []) for key in keys} def get_or_create_installation_id(self) -> str: config_file_manager = self.get_config_file_manager() @@ -95,15 +101,44 @@ def get_config_file_manager(self, scope: Optional[str] = None) -> ConfigFileMana return self.global_config_file_manager def get_scan_polling_timeout_in_seconds(self) -> int: - return int(self._get_value_from_environment_variables(SCAN_POLLING_TIMEOUT_IN_SECONDS_ENV_VAR_NAME, - DEFAULT_SCAN_POLLING_TIMEOUT_IN_SECONDS)) + return int( + self._get_value_from_environment_variables( + consts.SCAN_POLLING_TIMEOUT_IN_SECONDS_ENV_VAR_NAME, consts.DEFAULT_SCAN_POLLING_TIMEOUT_IN_SECONDS + ) + ) + + def get_sync_scan_timeout_in_seconds(self) -> int: + return int( + self._get_value_from_environment_variables( + consts.SYNC_SCAN_TIMEOUT_IN_SECONDS_ENV_VAR_NAME, consts.DEFAULT_SYNC_SCAN_TIMEOUT_IN_SECONDS + ) + ) + + def get_ai_remediation_timeout_in_seconds(self) -> int: + return int( + self._get_value_from_environment_variables( + consts.AI_REMEDIATION_TIMEOUT_IN_SECONDS_ENV_VAR_NAME, consts.DEFAULT_AI_REMEDIATION_TIMEOUT_IN_SECONDS + ) + ) + + def get_report_polling_timeout_in_seconds(self) -> int: + return int( + self._get_value_from_environment_variables( + consts.REPORT_POLLING_TIMEOUT_IN_SECONDS_ENV_VAR_NAME, consts.DEFAULT_REPORT_POLLING_TIMEOUT_IN_SECONDS + ) + ) def get_sca_pre_commit_timeout_in_seconds(self) -> int: - return int(self._get_value_from_environment_variables(SCA_PRE_COMMIT_TIMEOUT_IN_SECONDS_ENV_VAR_NAME, - DEFAULT_SCA_PRE_COMMIT_TIMEOUT_IN_SECONDS)) - - def get_pre_receive_max_commits_to_scan_count(self, command_scan_type: str) -> int: - max_commits = self._get_value_from_environment_variables(PRE_RECEIVE_MAX_COMMITS_TO_SCAN_COUNT_ENV_VAR_NAME) + return int( + self._get_value_from_environment_variables( + consts.SCA_PRE_COMMIT_TIMEOUT_IN_SECONDS_ENV_VAR_NAME, consts.DEFAULT_SCA_PRE_COMMIT_TIMEOUT_IN_SECONDS + ) + ) + + def _get_git_hook_max_commits_to_scan_count( + self, command_scan_type: str, env_var_name: str, default_count: int + ) -> int: + max_commits = self._get_value_from_environment_variables(env_var_name) if max_commits is not None: return int(max_commits) @@ -115,10 +150,24 @@ def get_pre_receive_max_commits_to_scan_count(self, command_scan_type: str) -> i if max_commits is not None: return max_commits - return DEFAULT_PRE_RECEIVE_MAX_COMMITS_TO_SCAN_COUNT + return default_count - def get_pre_receive_command_timeout(self, command_scan_type: str) -> int: - command_timeout = self._get_value_from_environment_variables(PRE_RECEIVE_COMMAND_TIMEOUT_ENV_VAR_NAME) + def get_pre_receive_max_commits_to_scan_count(self, command_scan_type: str) -> int: + return self._get_git_hook_max_commits_to_scan_count( + command_scan_type, + consts.PRE_RECEIVE_MAX_COMMITS_TO_SCAN_COUNT_ENV_VAR_NAME, + consts.DEFAULT_PRE_RECEIVE_MAX_COMMITS_TO_SCAN_COUNT, + ) + + def get_pre_push_max_commits_to_scan_count(self, command_scan_type: str) -> int: + return self._get_git_hook_max_commits_to_scan_count( + command_scan_type, + consts.PRE_PUSH_MAX_COMMITS_TO_SCAN_COUNT_ENV_VAR_NAME, + consts.DEFAULT_PRE_PUSH_MAX_COMMITS_TO_SCAN_COUNT, + ) + + def _get_git_hook_command_timeout(self, command_scan_type: str, env_var_name: str, default_timeout: int) -> int: + command_timeout = self._get_value_from_environment_variables(env_var_name) if command_timeout is not None: return int(command_timeout) @@ -130,26 +179,43 @@ def get_pre_receive_command_timeout(self, command_scan_type: str) -> int: if command_timeout is not None: return command_timeout - return DEFAULT_PRE_RECEIVE_COMMAND_TIMEOUT_IN_SECONDS + return default_timeout + + def get_pre_receive_command_timeout(self, command_scan_type: str) -> int: + return self._get_git_hook_command_timeout( + command_scan_type, + consts.PRE_RECEIVE_COMMAND_TIMEOUT_ENV_VAR_NAME, + consts.DEFAULT_PRE_RECEIVE_COMMAND_TIMEOUT_IN_SECONDS, + ) + + def get_pre_push_command_timeout(self, command_scan_type: str) -> int: + return self._get_git_hook_command_timeout( + command_scan_type, + consts.PRE_PUSH_COMMAND_TIMEOUT_ENV_VAR_NAME, + consts.DEFAULT_PRE_PUSH_COMMAND_TIMEOUT_IN_SECONDS, + ) def get_should_exclude_detections_in_deleted_lines(self, command_scan_type: str) -> bool: exclude_detections_in_deleted_lines = self._get_value_from_environment_variables( - EXCLUDE_DETECTIONS_IN_DELETED_LINES_ENV_VAR_NAME) + consts.EXCLUDE_DETECTIONS_IN_DELETED_LINES_ENV_VAR_NAME + ) if exclude_detections_in_deleted_lines is not None: return exclude_detections_in_deleted_lines.lower() in ('true', '1') - exclude_detections_in_deleted_lines = self.local_config_file_manager \ - .get_exclude_detections_in_deleted_lines(command_scan_type) + exclude_detections_in_deleted_lines = self.local_config_file_manager.get_exclude_detections_in_deleted_lines( + command_scan_type + ) if exclude_detections_in_deleted_lines is not None: return exclude_detections_in_deleted_lines - exclude_detections_in_deleted_lines = self.global_config_file_manager\ - .get_exclude_detections_in_deleted_lines(command_scan_type) + exclude_detections_in_deleted_lines = self.global_config_file_manager.get_exclude_detections_in_deleted_lines( + command_scan_type + ) if exclude_detections_in_deleted_lines is not None: return exclude_detections_in_deleted_lines - return DEFAULT_EXCLUDE_DETECTIONS_IN_DELETED_LINES + return consts.DEFAULT_EXCLUDE_DETECTIONS_IN_DELETED_LINES @staticmethod - def _get_value_from_environment_variables(env_var_name, default=None): + def _get_value_from_environment_variables(env_var_name: str, default: Optional[Any] = None) -> Optional[Any]: return os.getenv(env_var_name, default) diff --git a/cycode/cli/user_settings/credentials_manager.py b/cycode/cli/user_settings/credentials_manager.py index 67d31275..9522981b 100644 --- a/cycode/cli/user_settings/credentials_manager.py +++ b/cycode/cli/user_settings/credentials_manager.py @@ -1,50 +1,92 @@ import os from pathlib import Path +from typing import Optional -from cycode.cli.utils.yaml_utils import read_file -from cycode.cli.config import CYCODE_CLIENT_ID_ENV_VAR_NAME, CYCODE_CLIENT_SECRET_ENV_VAR_NAME +from cycode.cli.config import ( + CYCODE_CLIENT_ID_ENV_VAR_NAME, + CYCODE_CLIENT_SECRET_ENV_VAR_NAME, + CYCODE_ID_TOKEN_ENV_VAR_NAME, +) from cycode.cli.user_settings.base_file_manager import BaseFileManager +from cycode.cli.user_settings.jwt_creator import JwtCreator class CredentialsManager(BaseFileManager): - HOME_PATH: str = Path.home() CYCODE_HIDDEN_DIRECTORY: str = '.cycode' FILE_NAME: str = 'credentials.yaml' + CLIENT_ID_FIELD_NAME: str = 'cycode_client_id' CLIENT_SECRET_FIELD_NAME: str = 'cycode_client_secret' + ID_TOKEN_FIELD_NAME: str = 'cycode_id_token' + ACCESS_TOKEN_FIELD_NAME: str = 'cycode_access_token' + ACCESS_TOKEN_EXPIRES_IN_FIELD_NAME: str = 'cycode_access_token_expires_in' + ACCESS_TOKEN_CREATOR_FIELD_NAME: str = 'cycode_access_token_creator' - def get_credentials(self) -> (str, str): + def get_credentials(self) -> tuple[str, str]: client_id, client_secret = self.get_credentials_from_environment_variables() if client_id is not None and client_secret is not None: return client_id, client_secret return self.get_credentials_from_file() - def get_credentials_from_environment_variables(self) -> (str, str): + @staticmethod + def get_credentials_from_environment_variables() -> tuple[str, str]: client_id = os.getenv(CYCODE_CLIENT_ID_ENV_VAR_NAME) client_secret = os.getenv(CYCODE_CLIENT_SECRET_ENV_VAR_NAME) return client_id, client_secret - def get_credentials_from_file(self) -> (str, str): - credentials_filename = self.get_filename() - try: - file_content = read_file(credentials_filename) - except FileNotFoundError: - return None, None - + def get_credentials_from_file(self) -> tuple[Optional[str], Optional[str]]: + file_content = self.read_file() client_id = file_content.get(self.CLIENT_ID_FIELD_NAME) client_secret = file_content.get(self.CLIENT_SECRET_FIELD_NAME) return client_id, client_secret - def update_credentials_file(self, client_id: str, client_secret: str): - credentials = { - self.CLIENT_ID_FIELD_NAME: client_id, - self.CLIENT_SECRET_FIELD_NAME: client_secret - } + def get_oidc_credentials_from_file(self) -> tuple[Optional[str], Optional[str]]: + file_content = self.read_file() + client_id = file_content.get(self.CLIENT_ID_FIELD_NAME) + id_token = file_content.get(self.ID_TOKEN_FIELD_NAME) + return client_id, id_token + + def get_oidc_credentials(self) -> tuple[Optional[str], Optional[str]]: + client_id = os.getenv(CYCODE_CLIENT_ID_ENV_VAR_NAME) + id_token = os.getenv(CYCODE_ID_TOKEN_ENV_VAR_NAME) + + if client_id is not None and id_token is not None: + return client_id, id_token - filename = self.get_filename() - self.write_content_to_file(credentials) + return self.get_oidc_credentials_from_file() + + def update_oidc_credentials(self, client_id: str, id_token: str) -> None: + file_content_to_update = {self.CLIENT_ID_FIELD_NAME: client_id, self.ID_TOKEN_FIELD_NAME: id_token} + self.write_content_to_file(file_content_to_update) + + def update_credentials(self, client_id: str, client_secret: str) -> None: + file_content_to_update = {self.CLIENT_ID_FIELD_NAME: client_id, self.CLIENT_SECRET_FIELD_NAME: client_secret} + self.write_content_to_file(file_content_to_update) + + def get_access_token(self) -> tuple[Optional[str], Optional[float], Optional[JwtCreator]]: + file_content = self.read_file() + + access_token = file_content.get(self.ACCESS_TOKEN_FIELD_NAME) + expires_in = file_content.get(self.ACCESS_TOKEN_EXPIRES_IN_FIELD_NAME) + + creator = None + hashed_creator = file_content.get(self.ACCESS_TOKEN_CREATOR_FIELD_NAME) + if hashed_creator: + creator = JwtCreator(hashed_creator) + + return access_token, expires_in, creator + + def update_access_token( + self, access_token: Optional[str], expires_in: Optional[float], creator: Optional[JwtCreator] + ) -> None: + file_content_to_update = { + self.ACCESS_TOKEN_FIELD_NAME: access_token, + self.ACCESS_TOKEN_EXPIRES_IN_FIELD_NAME: expires_in, + self.ACCESS_TOKEN_CREATOR_FIELD_NAME: str(creator) if creator else None, + } + self.write_content_to_file(file_content_to_update) def get_filename(self) -> str: return os.path.join(self.HOME_PATH, self.CYCODE_HIDDEN_DIRECTORY, self.FILE_NAME) diff --git a/cycode/cli/user_settings/jwt_creator.py b/cycode/cli/user_settings/jwt_creator.py new file mode 100644 index 00000000..e3778f92 --- /dev/null +++ b/cycode/cli/user_settings/jwt_creator.py @@ -0,0 +1,24 @@ +from cycode.cli.utils.string_utils import hash_string_to_sha256 + +_SEPARATOR = '::' + + +def _get_hashed_creator(client_id: str, client_secret: str) -> str: + return hash_string_to_sha256(_SEPARATOR.join([client_id, client_secret])) + + +class JwtCreator: + def __init__(self, hashed_creator: str) -> None: + self._hashed_creator = hashed_creator + + def __str__(self) -> str: + return self._hashed_creator + + @classmethod + def create(cls, client_id: str, client_secret: str) -> 'JwtCreator': + return cls(_get_hashed_creator(client_id, client_secret)) + + def __eq__(self, other: 'JwtCreator') -> bool: + if not isinstance(other, JwtCreator): + return NotImplemented + return str(self) == str(other) diff --git a/cycode/cli/user_settings/user_settings_commands.py b/cycode/cli/user_settings/user_settings_commands.py deleted file mode 100644 index 24ebb0e9..00000000 --- a/cycode/cli/user_settings/user_settings_commands.py +++ /dev/null @@ -1,138 +0,0 @@ -import re -import os.path -from typing import Optional - -import click - -from cycode.cli.utils.string_utils import obfuscate_text, hash_string_to_sha256 -from cycode.cli.utils.path_utils import get_absolute_path -from cycode.cli.user_settings.credentials_manager import CredentialsManager -from cycode.cli.config import configuration_manager, config -from cycode.cli.consts import * -from cycode.cyclient import logger - -CREDENTIALS_UPDATED_SUCCESSFULLY_MESSAGE = 'Successfully configured CLI credentials!' -CREDENTIALS_ARE_SET_IN_ENVIRONMENT_VARIABLES_MESSAGE = 'Note that the credentials that already exist in environment' \ - ' variables (CYCODE_CLIENT_ID and CYCODE_CLIENT_SECRET) take' \ - ' precedent over these credentials; either update or remove ' \ - 'the environment variables.' -credentials_manager = CredentialsManager() - - -@click.command() -def set_credentials(): - """ Initial command to authenticate your CLI client with Cycode using client ID and client secret """ - click.echo(f'Update credentials in file ({credentials_manager.get_filename()})') - current_client_id, current_client_secret = credentials_manager.get_credentials_from_file() - client_id = _get_client_id_input(current_client_id) - client_secret = _get_client_secret_input(current_client_secret) - - if not _should_update_credentials(current_client_id, current_client_secret, client_id, client_secret): - return - - credentials_manager.update_credentials_file(client_id, client_secret) - click.echo(_get_credentials_update_result_message()) - - -@click.command() -@click.option("--by-value", type=click.STRING, required=False, - help="Ignore a specific value while scanning for secrets") -@click.option("--by-sha", type=click.STRING, required=False, - help='Ignore a specific SHA512 representation of a string while scanning for secrets') -@click.option("--by-path", type=click.STRING, required=False, - help='Avoid scanning a specific path. Need to specify scan type ') -@click.option("--by-rule", type=click.STRING, required=False, - help='Ignore scanning a specific secret rule ID/IaC rule ID. Need to specify scan type.') -@click.option("--by-package", type=click.STRING, required=False, - help='Ignore scanning a specific package version while running SCA scan. expected pattern - name@version') -@click.option('--scan-type', '-t', default='secret', - help=""" - \b - Specify the scan you wish to execute (secrets/iac), - the default is secrets - """, - type=click.Choice(config['scans']['supported_scans']), required=False) -@click.option('--global', '-g', 'is_global', is_flag=True, default=False, required=False, - help='Add an ignore rule and update it in the global .cycode config file') -def add_exclusions(by_value: str, by_sha: str, by_path: str, by_rule: str, by_package: str, scan_type: str, - is_global: bool): - """ Ignore a specific value, path or rule ID """ - if not by_value and not by_sha and not by_path and not by_rule and not by_package: - raise click.ClickException("ignore by type is missing") - - if by_value is not None: - if scan_type != SECRET_SCAN_TYPE: - raise click.ClickException("exclude by value is supported only for secret scan type") - exclusion_type = EXCLUSIONS_BY_VALUE_SECTION_NAME - exclusion_value = hash_string_to_sha256(by_value) - elif by_sha is not None: - if scan_type != SECRET_SCAN_TYPE: - raise click.ClickException("exclude by sha is supported only for secret scan type") - exclusion_type = EXCLUSIONS_BY_SHA_SECTION_NAME - exclusion_value = by_sha - elif by_path is not None: - absolute_path = get_absolute_path(by_path) - if not _is_path_to_ignore_exists(absolute_path): - raise click.ClickException("the provided path to ignore by is not exist") - exclusion_type = EXCLUSIONS_BY_PATH_SECTION_NAME - exclusion_value = get_absolute_path(absolute_path) - elif by_package is not None: - if scan_type != SCA_SCAN_TYPE: - raise click.ClickException("exclude by package is supported only for sca scan type") - if not _is_package_pattern_valid(by_package): - raise click.ClickException("wrong package pattern. should be name@version.") - exclusion_type = EXCLUSIONS_BY_PACKAGE_SECTION_NAME - exclusion_value = by_package - else: - exclusion_type = EXCLUSIONS_BY_RULE_SECTION_NAME - exclusion_value = by_rule - - configuration_scope = 'global' if is_global else 'local' - logger.debug('Adding ignore rule, %s', - {'configuration_scope': configuration_scope, 'exclusion_type': exclusion_type, - 'exclusion_value': exclusion_value}) - configuration_manager.add_exclusion(configuration_scope, scan_type, exclusion_type, exclusion_value) - - -def _get_client_id_input(current_client_id: str) -> str: - new_client_id = click.prompt(f'cycode client id [{_obfuscate_credential(current_client_id)}]', - default='', - show_default=False) - - return current_client_id if not new_client_id else new_client_id - - -def _get_client_secret_input(current_client_secret: str) -> str: - new_client_secret = click.prompt(f'cycode client secret [{_obfuscate_credential(current_client_secret)}]', - default='', - show_default=False) - return current_client_secret if not new_client_secret else new_client_secret - - -def _get_credentials_update_result_message(): - if not _are_credentials_exist_in_environment_variables(): - return CREDENTIALS_UPDATED_SUCCESSFULLY_MESSAGE - - return CREDENTIALS_UPDATED_SUCCESSFULLY_MESSAGE + ' ' + CREDENTIALS_ARE_SET_IN_ENVIRONMENT_VARIABLES_MESSAGE - - -def _are_credentials_exist_in_environment_variables(): - client_id, client_secret = credentials_manager.get_credentials_from_environment_variables() - return client_id is not None or client_secret is not None - - -def _should_update_credentials(current_client_id: str, current_client_secret: str, new_client_id: str, - new_client_secret: str) -> bool: - return current_client_id != new_client_id or current_client_secret != new_client_secret - - -def _obfuscate_credential(credential: Optional[str]) -> str: - return '' if not credential else obfuscate_text(credential) - - -def _is_path_to_ignore_exists(path: str) -> bool: - return os.path.exists(path) - - -def _is_package_pattern_valid(package: str) -> bool: - return re.search("^[^@]+@[^@]+$", package) is not None diff --git a/cycode/cli/utils/binary_utils.py b/cycode/cli/utils/binary_utils.py new file mode 100644 index 00000000..e61b7ddc --- /dev/null +++ b/cycode/cli/utils/binary_utils.py @@ -0,0 +1,72 @@ +_CONTROL_CHARS = b'\n\r\t\f\b' +_PRINTABLE_ASCII = _CONTROL_CHARS + bytes(range(32, 127)) +_PRINTABLE_HIGH_ASCII = bytes(range(127, 256)) + +# BOM signatures for encodings that legitimately contain null bytes +_BOM_ENCODINGS = ( + (b'\xff\xfe\x00\x00', 'utf-32-le'), + (b'\x00\x00\xfe\xff', 'utf-32-be'), + (b'\xff\xfe', 'utf-16-le'), + (b'\xfe\xff', 'utf-16-be'), +) + + +def _has_bom_encoding(bytes_to_check: bytes) -> bool: + """Check if bytes start with a BOM and can be decoded as that encoding.""" + for bom, encoding in _BOM_ENCODINGS: + if bytes_to_check.startswith(bom): + try: + bytes_to_check.decode(encoding) + return True + except (UnicodeDecodeError, LookupError): + pass + return False + + +def _is_decodable_as_utf8(bytes_to_check: bytes) -> bool: + """Try to decode bytes as UTF-8.""" + try: + bytes_to_check.decode('utf-8') + return True + except UnicodeDecodeError: + return False + + +def is_binary_string(bytes_to_check: bytes) -> bool: + """Check if a chunk of bytes appears to be binary content. + + Uses a simplified version of the Perl detection algorithm, matching + the structure of binaryornot's is_binary_string. + """ + if not bytes_to_check: + return False + + # Binary if control chars are > 30% of the string + low_chars = bytes_to_check.translate(None, _PRINTABLE_ASCII) + nontext_ratio1 = len(low_chars) / len(bytes_to_check) + + # Binary if high ASCII chars are < 5% of the string + high_chars = bytes_to_check.translate(None, _PRINTABLE_HIGH_ASCII) + nontext_ratio2 = len(high_chars) / len(bytes_to_check) + + is_likely_binary = (nontext_ratio1 > 0.3 and nontext_ratio2 < 0.05) or ( + nontext_ratio1 > 0.8 and nontext_ratio2 > 0.8 + ) + + # BOM-marked UTF-16/32 files legitimately contain null bytes. + # Check this first so they aren't misdetected as binary. + if _has_bom_encoding(bytes_to_check): + return False + + has_null_or_xff = b'\x00' in bytes_to_check or b'\xff' in bytes_to_check + + if is_likely_binary: + # Only let UTF-8 rescue data that doesn't contain null bytes. + # Null bytes are valid UTF-8 but almost never appear in real text files, + # whereas binary formats (e.g. .DS_Store) are full of them. + if has_null_or_xff: + return True + return not _is_decodable_as_utf8(bytes_to_check) + + # Null bytes or 0xff in otherwise normal-looking data indicate binary + return bool(has_null_or_xff) diff --git a/cycode/cli/utils/enum_utils.py b/cycode/cli/utils/enum_utils.py new file mode 100644 index 00000000..3280a5bb --- /dev/null +++ b/cycode/cli/utils/enum_utils.py @@ -0,0 +1,7 @@ +from enum import Enum + + +class AutoCountEnum(Enum): + @staticmethod + def _generate_next_value_(name: str, start: int, count: int, last_values: list[int]) -> int: + return count diff --git a/cycode/cli/utils/get_api_client.py b/cycode/cli/utils/get_api_client.py new file mode 100644 index 00000000..b69666d3 --- /dev/null +++ b/cycode/cli/utils/get_api_client.py @@ -0,0 +1,85 @@ +from typing import TYPE_CHECKING, Optional, Union + +import click + +from cycode.cli.user_settings.credentials_manager import CredentialsManager +from cycode.cyclient.client_creator import ( + create_ai_security_manager_client, + create_import_sbom_client, + create_report_client, + create_scan_client, +) + +if TYPE_CHECKING: + import typer + + from cycode.cyclient.ai_security_manager_client import AISecurityManagerClient + from cycode.cyclient.import_sbom_client import ImportSbomClient + from cycode.cyclient.report_client import ReportClient + from cycode.cyclient.scan_client import ScanClient + + +def _get_cycode_client( + create_client_func: callable, + client_id: Optional[str], + client_secret: Optional[str], + hide_response_log: bool, + id_token: Optional[str] = None, +) -> Union['ScanClient', 'ReportClient', 'ImportSbomClient', 'AISecurityManagerClient']: + if client_id and id_token: + return create_client_func(client_id, None, hide_response_log, id_token) + + if not client_id or not id_token: + oidc_client_id, oidc_id_token = _get_configured_oidc_credentials() + if oidc_client_id and oidc_id_token: + return create_client_func(oidc_client_id, None, hide_response_log, oidc_id_token) + if oidc_id_token and not oidc_client_id: + raise click.ClickException('Cycode client id needed for OIDC authentication.') + + if not client_id or not client_secret: + client_id, client_secret = _get_configured_credentials() + if not client_id: + raise click.ClickException('Cycode client id needed.') + if not client_secret: + raise click.ClickException('Cycode client secret is needed.') + + return create_client_func(client_id, client_secret, hide_response_log, None) + + +def get_scan_cycode_client(ctx: 'typer.Context') -> 'ScanClient': + client_id = ctx.obj.get('client_id') + client_secret = ctx.obj.get('client_secret') + id_token = ctx.obj.get('id_token') + hide_response_log = not ctx.obj.get('show_secret', False) + return _get_cycode_client(create_scan_client, client_id, client_secret, hide_response_log, id_token) + + +def get_report_cycode_client(ctx: 'typer.Context', hide_response_log: bool = True) -> 'ReportClient': + client_id = ctx.obj.get('client_id') + client_secret = ctx.obj.get('client_secret') + id_token = ctx.obj.get('id_token') + return _get_cycode_client(create_report_client, client_id, client_secret, hide_response_log, id_token) + + +def get_import_sbom_cycode_client(ctx: 'typer.Context', hide_response_log: bool = True) -> 'ImportSbomClient': + client_id = ctx.obj.get('client_id') + client_secret = ctx.obj.get('client_secret') + id_token = ctx.obj.get('id_token') + return _get_cycode_client(create_import_sbom_client, client_id, client_secret, hide_response_log, id_token) + + +def get_ai_security_manager_client(ctx: 'typer.Context', hide_response_log: bool = True) -> 'AISecurityManagerClient': + client_id = ctx.obj.get('client_id') + client_secret = ctx.obj.get('client_secret') + id_token = ctx.obj.get('id_token') + return _get_cycode_client(create_ai_security_manager_client, client_id, client_secret, hide_response_log, id_token) + + +def _get_configured_credentials() -> tuple[str, str]: + credentials_manager = CredentialsManager() + return credentials_manager.get_credentials() + + +def _get_configured_oidc_credentials() -> tuple[Optional[str], Optional[str]]: + credentials_manager = CredentialsManager() + return credentials_manager.get_oidc_credentials() diff --git a/cycode/cli/utils/git_proxy.py b/cycode/cli/utils/git_proxy.py new file mode 100644 index 00000000..beaafdd0 --- /dev/null +++ b/cycode/cli/utils/git_proxy.py @@ -0,0 +1,97 @@ +import types +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING, Optional + +_GIT_ERROR_MESSAGE = """ +Cycode CLI needs the Git executable to be installed on the system. +Git executable must be available in the PATH. +Git 1.7.x or newer is required. +You can help Cycode CLI to locate the Git executable +by setting the GIT_PYTHON_GIT_EXECUTABLE= environment variable. +""".strip().replace('\n', ' ') + +try: + import git +except ImportError: + git = None + +if TYPE_CHECKING: + from git import PathLike, Repo + + +class GitProxyError(Exception): + pass + + +class _AbstractGitProxy(ABC): + @abstractmethod + def get_repo(self, path: Optional['PathLike'] = None, *args, **kwargs) -> 'Repo': ... + + @abstractmethod + def get_null_tree(self) -> object: ... + + @abstractmethod + def get_invalid_git_repository_error(self) -> type[BaseException]: ... + + @abstractmethod + def get_git_command_error(self) -> type[BaseException]: ... + + +class _DummyGitProxy(_AbstractGitProxy): + def get_repo(self, path: Optional['PathLike'] = None, *args, **kwargs) -> 'Repo': + raise RuntimeError(_GIT_ERROR_MESSAGE) + + def get_null_tree(self) -> object: + raise RuntimeError(_GIT_ERROR_MESSAGE) + + def get_invalid_git_repository_error(self) -> type[BaseException]: + return GitProxyError + + def get_git_command_error(self) -> type[BaseException]: + return GitProxyError + + +class _GitProxy(_AbstractGitProxy): + def get_repo(self, path: Optional['PathLike'] = None, *args, **kwargs) -> 'Repo': + return git.Repo(path, *args, **kwargs) + + def get_null_tree(self) -> object: + return git.NULL_TREE + + def get_invalid_git_repository_error(self) -> type[BaseException]: + return git.InvalidGitRepositoryError + + def get_git_command_error(self) -> type[BaseException]: + return git.GitCommandError + + +def get_git_proxy(git_module: Optional[types.ModuleType]) -> _AbstractGitProxy: + return _GitProxy() if git_module else _DummyGitProxy() + + +class GitProxyManager(_AbstractGitProxy): + """We are using this manager for easy unit testing and mocking of the git module.""" + + def __init__(self) -> None: + self._git_proxy = get_git_proxy(git) + + def _set_dummy_git_proxy(self) -> None: + self._git_proxy = _DummyGitProxy() + + def _set_git_proxy(self) -> None: + self._git_proxy = _GitProxy() + + def get_repo(self, path: Optional['PathLike'] = None, *args, **kwargs) -> 'Repo': + return self._git_proxy.get_repo(path, *args, **kwargs) + + def get_null_tree(self) -> object: + return self._git_proxy.get_null_tree() + + def get_invalid_git_repository_error(self) -> type[BaseException]: + return self._git_proxy.get_invalid_git_repository_error() + + def get_git_command_error(self) -> type[BaseException]: + return self._git_proxy.get_git_command_error() + + +git_proxy = GitProxyManager() diff --git a/cycode/cli/utils/ignore_utils.py b/cycode/cli/utils/ignore_utils.py new file mode 100644 index 00000000..98126658 --- /dev/null +++ b/cycode/cli/utils/ignore_utils.py @@ -0,0 +1,478 @@ +# Copyright (C) 2017 Jelmer Vernooij +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Modified from https://github.com/jelmer/dulwich/blob/master/dulwich/ignore.py + +# Copyright 2020 Ben Kehoe +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Modified from https://github.com/benkehoe/ignorelib/blob/main/ignorelib.py + +"""Parsing of ignore files according to gitignore rules. + +For details for the matching rules, see https://git-scm.com/docs/gitignore +""" + +import contextlib +import os.path +import re +from collections.abc import Generator, Iterable +from os import PathLike +from typing import ( + Any, + BinaryIO, + Optional, + Union, +) + + +def _translate_segment(segment: bytes) -> bytes: # noqa: C901 + if segment == b'*': + return b'[^/]+' + res = b'' + i, n = 0, len(segment) + while i < n: + c = segment[i : i + 1] + i = i + 1 + if c == b'*': + res += b'[^/]*' + elif c == b'?': + res += b'[^/]' + elif c == b'\\': + res += re.escape(segment[i : i + 1]) + i += 1 + elif c == b'[': + j = i + if j < n and segment[j : j + 1] == b'!': + j = j + 1 + if j < n and segment[j : j + 1] == b']': + j = j + 1 + while j < n and segment[j : j + 1] != b']': + j = j + 1 + if j >= n: + res += b'\\[' + else: + stuff = segment[i:j].replace(b'\\', b'\\\\') + i = j + 1 + if stuff.startswith(b'!'): + stuff = b'^' + stuff[1:] + elif stuff.startswith(b'^'): + stuff = b'\\' + stuff + res += b'[' + stuff + b']' + else: + res += re.escape(c) + return res + + +def translate(pat: bytes) -> bytes: + """Translate a shell PATTERN to a regular expression. + + There is no way to quote meta-characters. + + Originally copied from fnmatch in Python 2.7, but modified for Dulwich + to cope with features in Git ignore patterns. + """ + res = b'(?ms)' + + if b'/' not in pat[:-1]: + # If there's no slash, this is a filename-based match + res += b'(.*/)?' + + if pat.startswith(b'**/'): + # Leading **/ + pat = pat[2:] + res += b'(.*/)?' + + if pat.startswith(b'/'): + pat = pat[1:] + + for i, segment in enumerate(pat.split(b'/')): + if segment == b'**': + res += b'(/.*)?' + continue + res += (re.escape(b'/') if i > 0 else b'') + _translate_segment(segment) + + if not pat.endswith(b'/'): + res += b'/?' + + return res + b'\\Z' + + +def read_ignore_patterns(f: BinaryIO) -> Iterable[bytes]: + """Read a git ignore file. + + Args: + f: File-like object to read from + Returns: List of patterns + + """ + for line in f: + line = line.rstrip(b'\r\n') + + # Ignore blank lines, they're used for readability. + if not line.strip(): + continue + + if line.startswith(b'#'): + # Comment + continue + + # Trailing spaces are ignored unless they are quoted with a backslash. + while line.endswith(b' ') and not line.endswith(b'\\ '): + line = line[:-1] + line = line.replace(b'\\ ', b' ') + + yield line + + +def match_pattern(path: bytes, pattern: bytes, ignore_case: bool = False) -> bool: + """Match a gitignore-style pattern against a path. + + Args: + path: Path to match + pattern: Pattern to match + ignore_case: Whether to do case-sensitive matching + Returns: + bool indicating whether the pattern matched + + """ + return Pattern(pattern, ignore_case).match(path) + + +class Pattern: + """A single ignore pattern.""" + + def __init__(self, pattern: bytes, ignore_case: bool = False) -> None: + self.pattern = pattern + self.ignore_case = ignore_case + if pattern[0:1] == b'!': + self.is_exclude = False + pattern = pattern[1:] + else: + if pattern[0:1] == b'\\': + pattern = pattern[1:] + self.is_exclude = True + flags = 0 + if self.ignore_case: + flags = re.IGNORECASE + self._re = re.compile(translate(pattern), flags) + + def __bytes__(self) -> bytes: + return self.pattern + + def __str__(self) -> str: + return os.fsdecode(self.pattern) + + def __eq__(self, other: object) -> bool: + return isinstance(other, type(self)) and self.pattern == other.pattern and self.ignore_case == other.ignore_case + + def __repr__(self) -> str: + return f'{type(self).__name__}({self.pattern!r}, {self.ignore_case!r})' + + def match(self, path: bytes) -> bool: + """Try to match a path against this ignore pattern. + + Args: + path: Path to match (relative to ignore location) + Returns: boolean + + """ + return bool(self._re.match(path)) + + +class IgnoreFilter: + def __init__( + self, + patterns: Iterable[Union[str, bytes]], + ignore_case: bool = False, + path: Optional[Union[PathLike, str]] = None, + ) -> None: + if hasattr(path, '__fspath__'): + path = path.__fspath__() + self._patterns = [] # type: List[Pattern] + self._ignore_case = ignore_case + self._path = path + for pattern in patterns: + self.append_pattern(pattern) + + def to_dict(self) -> dict[str, Any]: + d = { + 'patterns': [str(p) for p in self._patterns], + 'ignore_case': self._ignore_case, + } + path = getattr(self, '_path', None) + if path: + d['path'] = path + return d + + def append_pattern(self, pattern: Union[str, bytes]) -> None: + """Add a pattern to the set.""" + if isinstance(pattern, str): + pattern = bytes(pattern, 'utf-8') + self._patterns.append(Pattern(pattern, self._ignore_case)) + + def find_matching(self, path: Union[bytes, str]) -> Iterable[Pattern]: + """Yield all matching patterns for path. + + Args: + path: Path to match + Returns: + Iterator over iterators + + """ + if not isinstance(path, bytes): + path = os.fsencode(path) + for pattern in self._patterns: + if pattern.match(path): + yield pattern + + def is_ignored(self, path: Union[bytes, str]) -> Optional[bool]: + """Check whether a path is ignored. + + For directories, include a trailing slash. + + Returns: status is None if file is not mentioned, True if it is + included, False if it is explicitly excluded. + """ + if hasattr(path, '__fspath__'): + path = path.__fspath__() + status = None + for pattern in self.find_matching(path): + status = pattern.is_exclude + return status + + @classmethod + def from_path(cls, path: Union[PathLike, str], ignore_case: bool = False) -> 'IgnoreFilter': + if hasattr(path, '__fspath__'): + path = path.__fspath__() + with open(path, 'rb') as f: + return cls(read_ignore_patterns(f), ignore_case, path=path) + + def __repr__(self) -> str: + path = getattr(self, '_path', None) + if path is not None: + return f'{type(self).__name__}.from_path({path!r})' + return f'<{type(self).__name__}>' + + +class IgnoreFilterManager: + """Ignore file manager.""" + + def __init__( + self, + path: str, + global_filters: list[IgnoreFilter], + ignore_file_name: Optional[str] = None, + ignore_case: bool = False, + ) -> None: + if hasattr(path, '__fspath__'): + path = path.__fspath__() + self._path_filters = {} # type: Dict[str, Optional[IgnoreFilter]] + self._top_path = path + self._global_filters = global_filters + + self._ignore_file_name = ignore_file_name + if self._ignore_file_name is None: + self._ignore_file_name = '.gitignore' + + self._ignore_case = ignore_case + + def __repr__(self) -> str: + return f'{type(self).__name__}({self._top_path}, {self._global_filters!r}, {self._ignore_case!r})' + + def to_dict(self, include_path_filters: bool = True) -> dict[str, Any]: + d = { + 'path': self._top_path, + 'global_filters': [f.to_dict() for f in self._global_filters], + 'ignore_case': self._ignore_case, + } + if include_path_filters: + d['path_filters'] = {path: f.to_dict() for path, f in self._path_filters.items() if f is not None} + return d + + @property + def path(self) -> str: + return self._top_path + + @property + def ignore_file_name(self) -> Optional[str]: + return self._ignore_file_name + + @property + def ignore_case(self) -> bool: + return self._ignore_case + + def _load_path(self, path: str) -> Optional[IgnoreFilter]: + try: + return self._path_filters[path] + except KeyError: + pass + + if not self._ignore_file_name: + self._path_filters[path] = None + else: + p = os.path.join(self._top_path, path, self._ignore_file_name) + try: + self._path_filters[path] = IgnoreFilter.from_path(p, self._ignore_case) + except OSError: + self._path_filters[path] = None + return self._path_filters[path] + + def _find_matching(self, path: str) -> Iterable[Pattern]: + """Find matching patterns for path. + + Args: + path: Path to check + Returns: + Iterator over Pattern instances + + """ + if os.path.isabs(path): + raise ValueError(f'{path} is an absolute path') + filters = [(0, f) for f in self._global_filters] + if os.path.sep != '/': + path = path.replace(os.path.sep, '/') + parts = path.split('/') + matches = [] + for i in range(len(parts) + 1): + dirname = '/'.join(parts[:i]) + for s, f in filters: + relpath = '/'.join(parts[s:i]) + if i < len(parts): + # Paths leading up to the final part are all directories, + # so need a trailing slash. + relpath += '/' + matches += list(f.find_matching(relpath)) + ignore_filter = self._load_path(dirname) + if ignore_filter is not None: + filters.insert(0, (i, ignore_filter)) + return iter(matches) + + def is_ignored(self, path: str) -> Optional[bool]: + """Check whether a path is ignored. + + Args: + path: Path to check, relative to the IgnoreFilterManager path + Returns: + True if the path matches an ignore pattern, + False if the path is explicitly not ignored, + or None if the file does not match any patterns. + + """ + if hasattr(path, '__fspath__'): + path = path.__fspath__() + matches = list(self._find_matching(path)) + if matches: + return matches[-1].is_exclude + return None + + def walk_with_ignored( + self, **kwargs + ) -> Generator[tuple[str, list[str], list[str], list[str], list[str]], None, None]: + """Wrap os.walk() and also return lists of ignored directories and files. + + Yields tuples: (dirpath, included_dirnames, included_filenames, ignored_dirnames, ignored_filenames) + """ + for dirpath, dirnames, filenames in os.walk(self.path, topdown=True, **kwargs): + rel_dirpath = '' if dirpath == self.path else os.path.relpath(dirpath, self.path) + + original_dirnames = list(dirnames) + included_dirnames = [] + ignored_dirnames = [] + for d in original_dirnames: + if self.is_ignored(os.path.join(rel_dirpath, d)): + ignored_dirnames.append(d) + else: + included_dirnames.append(d) + + # decrease recursion depth of os.walk() by ignoring subdirectories because of topdown=True + # slicing ([:]) is mandatory to change dict in-place! + dirnames[:] = included_dirnames + + included_filenames = [] + ignored_filenames = [] + for f in filenames: + if self.is_ignored(os.path.join(rel_dirpath, f)): + ignored_filenames.append(f) + else: + included_filenames.append(f) + + yield dirpath, dirnames, included_filenames, ignored_dirnames, ignored_filenames + + @classmethod + def build( + cls, + path: str, + global_ignore_file_paths: Optional[Iterable[str]] = None, + global_patterns: Optional[Iterable[Union[str, bytes]]] = None, + ignore_file_name: Optional[str] = None, + ignore_case: bool = False, + ) -> 'IgnoreFilterManager': + """Create a IgnoreFilterManager from patterns and paths. + + Args: + path: The root path for ignore checks. + global_ignore_file_paths: A list of file paths to load patterns from. + Relative paths are relative to the IgnoreFilterManager path, not + the current directory. + global_patterns: Global patterns to ignore. + ignore_file_name: The per-directory ignore file name. + ignore_case: Whether to ignore case in matching. + + Returns: + A `IgnoreFilterManager` object + + """ + if not global_ignore_file_paths: + global_ignore_file_paths = [] + if not global_patterns: + global_patterns = [] + + global_ignore_file_paths.extend( + [ + os.path.join('.git', 'info', 'exclude'), # relative to an input path, so within the repo + os.path.expanduser(os.path.join('~', '.config', 'git', 'ignore')), # absolute + ] + ) + + if hasattr(path, '__fspath__'): + path = path.__fspath__() + + global_filters = [] + for p in global_ignore_file_paths: + if hasattr(p, '__fspath__'): + p = p.__fspath__() + + p = os.path.expanduser(p) + if not os.path.isabs(p): + p = os.path.join(path, p) + + with contextlib.suppress(IOError): + global_filters.append(IgnoreFilter.from_path(p)) + + if global_patterns: + global_filters.append(IgnoreFilter(global_patterns)) + + return cls(path, global_filters=global_filters, ignore_file_name=ignore_file_name, ignore_case=ignore_case) diff --git a/cycode/cli/utils/jwt_utils.py b/cycode/cli/utils/jwt_utils.py new file mode 100644 index 00000000..c87b7c48 --- /dev/null +++ b/cycode/cli/utils/jwt_utils.py @@ -0,0 +1,19 @@ +from typing import Optional + +import jwt + +_JWT_PAYLOAD_POSSIBLE_USER_ID_FIELD_NAMES = ('userId', 'internalId', 'token-user-id') + + +def get_user_and_tenant_ids_from_access_token(access_token: str) -> tuple[Optional[str], Optional[str]]: + payload = jwt.decode(access_token, options={'verify_signature': False}) + + user_id = None + for field in _JWT_PAYLOAD_POSSIBLE_USER_ID_FIELD_NAMES: + user_id = payload.get(field) + if user_id: + break + + tenant_id = payload.get('tenantId') + + return user_id, tenant_id diff --git a/cycode/cli/utils/path_utils.py b/cycode/cli/utils/path_utils.py index d7c4478c..c2d59805 100644 --- a/cycode/cli/utils/path_utils.py +++ b/cycode/cli/utils/path_utils.py @@ -1,26 +1,18 @@ -from typing import Iterable, List, Optional -import pathspec +import json import os -from pathlib import Path -from binaryornot.check import is_binary +from functools import cache +from typing import TYPE_CHECKING, AnyStr, Optional, Union +import typer -def get_relevant_files_in_path(path: str, exclude_patterns: Iterable[str]) -> List[str]: - absolute_path = get_absolute_path(path) - if not os.path.isfile(absolute_path) and not os.path.isdir(absolute_path): - raise FileNotFoundError(f'the specified path was not found, path: {path}') +from cycode.cli.logger import logger +from cycode.cli.utils.binary_utils import is_binary_string - if os.path.isfile(absolute_path): - return [absolute_path] - - directory_files_paths = _get_all_existing_files_in_directory(absolute_path) - file_paths = set({str(file_path) for file_path in directory_files_paths}) - spec = pathspec.PathSpec.from_lines(pathspec.patterns.GitWildMatchPattern, exclude_patterns) - exclude_file_paths = set(spec.match_files(file_paths)) - - return [file_path for file_path in (file_paths - exclude_file_paths) if os.path.isfile(file_path)] +if TYPE_CHECKING: + from os import PathLike +@cache def is_sub_path(path: str, sub_path: str) -> bool: try: common_path = os.path.commonpath([get_absolute_path(path), get_absolute_path(sub_path)]) @@ -36,8 +28,28 @@ def get_absolute_path(path: str) -> str: return os.path.abspath(path) +def _get_starting_chunk(filename: str, length: int = 1024) -> Optional[bytes]: + # We are using our own implementation of get_starting_chunk + # because the original one from binaryornot uses print()... + + try: + with open(filename, 'rb') as f: + return f.read(length) + except OSError as e: + logger.debug('Failed to read the starting chunk from file: %s', filename, exc_info=e) + + return None + + def is_binary_file(filename: str) -> bool: - return is_binary(filename) + # Check if the file extension is in a list of known binary types + binary_extensions = ('.pyc',) + if filename.endswith(binary_extensions): + return True + + # Check if the starting chunk is a binary string + chunk = _get_starting_chunk(filename) + return is_binary_string(chunk) def get_file_size(filename: str) -> int: @@ -48,12 +60,7 @@ def get_path_by_os(filename: str) -> str: return filename.replace('/', os.sep) -def _get_all_existing_files_in_directory(path: str): - directory = Path(path) - return directory.rglob(r"*") - - -def is_path_exists(path: str): +def is_path_exists(path: str) -> bool: return os.path.exists(path) @@ -61,14 +68,54 @@ def get_file_dir(path: str) -> str: return os.path.dirname(path) +def get_immediate_subdirectories(path: str) -> list[str]: + return [f.name for f in os.scandir(path) if f.is_dir()] + + def join_paths(path: str, filename: str) -> str: return os.path.join(path, filename) -def get_file_content(file_path: str) -> Optional[str]: +def get_file_content(file_path: Union[str, 'PathLike']) -> Optional[AnyStr]: try: - with open(file_path, "r", encoding="utf-8") as f: - content = f.read() - return content - except FileNotFoundError: + with open(file_path, encoding='UTF-8') as f: + return f.read() + except (FileNotFoundError, UnicodeDecodeError): return None + except PermissionError: + logger.warn('Permission denied to read the file: %s', file_path) + + +def load_json(txt: str) -> Optional[dict]: + try: + return json.loads(txt) + except json.JSONDecodeError: + return None + + +def change_filename_extension(filename: str, extension: str) -> str: + base_name, _ = os.path.splitext(filename) + return f'{base_name}.{extension}' + + +def concat_unique_id(filename: str, unique_id: str) -> str: + if filename.startswith(os.sep): + # remove leading slash to join the path correctly + filename = filename[len(os.sep) :] + + return os.path.join(unique_id, filename) + + +def get_path_from_context(ctx: typer.Context) -> Optional[str]: + path = ctx.params.get('path') + if path is None and 'paths' in ctx.params: + path = ctx.params['paths'][0] + return path + + +def normalize_file_path(path: str) -> str: + if path.startswith('/'): + return path[1:] + if path.startswith('./'): + return path[2:] + return path diff --git a/cycode/cli/utils/progress_bar.py b/cycode/cli/utils/progress_bar.py new file mode 100644 index 00000000..7c2de487 --- /dev/null +++ b/cycode/cli/utils/progress_bar.py @@ -0,0 +1,274 @@ +from abc import ABC, abstractmethod +from enum import auto +from typing import NamedTuple, Optional + +from rich.progress import BarColumn, Progress, SpinnerColumn, TaskProgressColumn, TextColumn, TimeElapsedColumn + +from cycode.cli.console import console +from cycode.cli.utils.enum_utils import AutoCountEnum +from cycode.logger import get_logger + +# use LOGGING_LEVEL=DEBUG env var to see debug logs of this module +logger = get_logger('Progress Bar', control_level_in_runtime=False) + + +class ProgressBarSection(AutoCountEnum): + def has_next(self) -> bool: + return self.value < len(type(self)) - 1 + + def next(self) -> 'ProgressBarSection': + return type(self)(self.value + 1) + + +class ProgressBarSectionInfo(NamedTuple): + section: ProgressBarSection + label: str + start_percent: int + stop_percent: int + initial: bool = False + + +_PROGRESS_BAR_LENGTH = 100 +_PROGRESS_BAR_COLUMNS = ( + SpinnerColumn(), + TextColumn('[progress.description]{task.description}'), + TextColumn('{task.fields[right_side_label]}'), + BarColumn(bar_width=None), + TaskProgressColumn(), + TimeElapsedColumn(), +) + +ProgressBarSections = dict[ProgressBarSection, ProgressBarSectionInfo] + + +class ScanProgressBarSection(ProgressBarSection): + PREPARE_LOCAL_FILES = auto() + SCAN = auto() + GENERATE_REPORT = auto() + + +SCAN_PROGRESS_BAR_SECTIONS: ProgressBarSections = { + ScanProgressBarSection.PREPARE_LOCAL_FILES: ProgressBarSectionInfo( + ScanProgressBarSection.PREPARE_LOCAL_FILES, 'Prepare local files', start_percent=0, stop_percent=5, initial=True + ), + ScanProgressBarSection.SCAN: ProgressBarSectionInfo( + ScanProgressBarSection.SCAN, 'Scan in progress', start_percent=5, stop_percent=95 + ), + ScanProgressBarSection.GENERATE_REPORT: ProgressBarSectionInfo( + ScanProgressBarSection.GENERATE_REPORT, 'Generate report', start_percent=95, stop_percent=100 + ), +} + + +class SbomReportProgressBarSection(ProgressBarSection): + PREPARE_LOCAL_FILES = auto() + GENERATION = auto() + RECEIVE_REPORT = auto() + + +SBOM_REPORT_PROGRESS_BAR_SECTIONS: ProgressBarSections = { + SbomReportProgressBarSection.PREPARE_LOCAL_FILES: ProgressBarSectionInfo( + SbomReportProgressBarSection.PREPARE_LOCAL_FILES, + 'Prepare local files', + start_percent=0, + stop_percent=30, + initial=True, + ), + SbomReportProgressBarSection.GENERATION: ProgressBarSectionInfo( + SbomReportProgressBarSection.GENERATION, 'Report generation in progress', start_percent=30, stop_percent=90 + ), + SbomReportProgressBarSection.RECEIVE_REPORT: ProgressBarSectionInfo( + SbomReportProgressBarSection.RECEIVE_REPORT, 'Receive report', start_percent=90, stop_percent=100 + ), +} + + +def _get_initial_section(progress_bar_sections: ProgressBarSections) -> ProgressBarSectionInfo: + for section in progress_bar_sections.values(): + if section.initial: + return section + + raise ValueError('No initial section found') + + +class BaseProgressBar(ABC): + @abstractmethod + def __init__(self, *args, **kwargs) -> None: + pass + + @abstractmethod + def start(self) -> None: ... + + @abstractmethod + def stop(self) -> None: ... + + @abstractmethod + def set_section_length(self, section: 'ProgressBarSection', length: int = 0) -> None: ... + + @abstractmethod + def update(self, section: 'ProgressBarSection') -> None: ... + + @abstractmethod + def update_right_side_label(self, label: Optional[str] = None) -> None: ... + + +class DummyProgressBar(BaseProgressBar): + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + + def start(self) -> None: + pass + + def stop(self) -> None: + pass + + def set_section_length(self, section: 'ProgressBarSection', length: int = 0) -> None: + pass + + def update(self, section: 'ProgressBarSection') -> None: + pass + + def update_right_side_label(self, label: Optional[str] = None) -> None: + pass + + +class CompositeProgressBar(BaseProgressBar): + def __init__(self, progress_bar_sections: ProgressBarSections) -> None: + super().__init__() + + self._progress_bar_sections = progress_bar_sections + + self._section_lengths: dict[ProgressBarSection, int] = {} + self._section_values: dict[ProgressBarSection, int] = {} + + self._current_section_value = 0 + self._current_section: ProgressBarSectionInfo = _get_initial_section(self._progress_bar_sections) + self._current_right_side_label = '' + + self._progress_bar = Progress(*_PROGRESS_BAR_COLUMNS, console=console, refresh_per_second=5, transient=True) + self._progress_bar_task_id = self._progress_bar.add_task( + description=self._current_section.label, + total=_PROGRESS_BAR_LENGTH, + right_side_label=self._current_right_side_label, + ) + + def _progress_bar_update(self, advance: int = 0) -> None: + self._progress_bar.update( + self._progress_bar_task_id, + advance=advance, + description=self._current_section.label, + right_side_label=self._current_right_side_label, + refresh=True, + ) + + def start(self) -> None: + self._progress_bar.start() + + def stop(self) -> None: + self._progress_bar.stop() + + def set_section_length(self, section: 'ProgressBarSection', length: int = 0) -> None: + logger.debug('Calling set_section_length, %s', {'section': str(section), 'length': length}) + self._section_lengths[section] = length + + if length == 0: + self._skip_section(section) + else: + self._maybe_update_current_section() + + def _get_section_length(self, section: 'ProgressBarSection') -> int: + section_info = self._progress_bar_sections[section] + return section_info.stop_percent - section_info.start_percent + + def _skip_section(self, section: 'ProgressBarSection') -> None: + self._progress_bar_update(self._get_section_length(section)) + self._maybe_update_current_section() + + def _increment_section_value(self, section: 'ProgressBarSection', value: int) -> None: + self._section_values[section] = self._section_values.get(section, 0) + value + logger.debug( + 'Calling _increment_section_value: %s +%s. %s/%s', + section, + value, + self._section_values[section], + self._section_lengths[section], + ) + + def _rerender_progress_bar(self) -> None: + """Use to update label right after changing the progress bar section.""" + self._progress_bar_update() + + def _increment_progress(self, section: 'ProgressBarSection') -> None: + increment_value = self._get_increment_progress_value(section) + + self._current_section_value += increment_value + self._progress_bar_update(increment_value) + + def _maybe_update_current_section(self) -> None: + if not self._current_section.section.has_next(): + return + + max_val = self._section_lengths.get(self._current_section.section, 0) + cur_val = self._section_values.get(self._current_section.section, 0) + if cur_val >= max_val: + next_section = self._progress_bar_sections[self._current_section.section.next()] + logger.debug( + 'Calling _update_current_section: %s -> %s', self._current_section.section, next_section.section + ) + + self._current_section = next_section + self._current_section_value = 0 + self._rerender_progress_bar() + + def _get_increment_progress_value(self, section: 'ProgressBarSection') -> int: + max_val = self._section_lengths[section] + cur_val = self._section_values[section] + + expected_value = round(self._get_section_length(section) * (cur_val / max_val)) + + return expected_value - self._current_section_value + + def update(self, section: 'ProgressBarSection', value: int = 1) -> None: + if section not in self._section_lengths: + raise ValueError(f'{section} section is not initialized. Call set_section_length() first.') + if section is not self._current_section.section: + raise ValueError( + f'Previous section is not completed yet. Complete {self._current_section.section} section first.' + ) + + self._increment_section_value(section, value) + self._increment_progress(section) + self._maybe_update_current_section() + + def update_right_side_label(self, label: Optional[str] = None) -> None: + self._current_right_side_label = f'({label})' if label else '' + self._progress_bar_update() + + +def get_progress_bar(*, hidden: bool, sections: ProgressBarSections) -> BaseProgressBar: + if hidden: + return DummyProgressBar() + + return CompositeProgressBar(sections) + + +if __name__ == '__main__': + # TODO(MarshalX): cover with tests and remove this code + import random + import time + + bar = get_progress_bar(hidden=False, sections=SCAN_PROGRESS_BAR_SECTIONS) + bar.start() + + for bar_section in ScanProgressBarSection: + section_capacity = random.randint(500, 1000) # noqa: S311 + bar.set_section_length(bar_section, section_capacity) + + for _i in range(section_capacity): + time.sleep(0.01) + bar.update_right_side_label(f'{bar_section} {_i}/{section_capacity}') + bar.update(bar_section) + + bar.update_right_side_label() + + bar.stop() diff --git a/cycode/cli/utils/scan_batch.py b/cycode/cli/utils/scan_batch.py new file mode 100644 index 00000000..97e58bc7 --- /dev/null +++ b/cycode/cli/utils/scan_batch.py @@ -0,0 +1,145 @@ +import os +from multiprocessing.pool import ThreadPool +from typing import TYPE_CHECKING, Callable + +from cycode.cli import consts +from cycode.cli.models import Document +from cycode.cli.utils.progress_bar import ScanProgressBarSection +from cycode.logger import get_logger + +if TYPE_CHECKING: + from cycode.cli.models import CliError, LocalScanResult + from cycode.cli.utils.progress_bar import BaseProgressBar + + +logger = get_logger('Batching') + + +def _get_max_batch_size(scan_type: str) -> int: + logger.debug( + 'You can customize the batch size by setting the environment variable "%s"', + consts.SCAN_BATCH_MAX_SIZE_IN_BYTES_ENV_VAR_NAME, + ) + + custom_size = os.environ.get(consts.SCAN_BATCH_MAX_SIZE_IN_BYTES_ENV_VAR_NAME) + if custom_size: + logger.debug('Custom batch size is set, %s', {'custom_size': custom_size}) + return int(custom_size) + + return consts.SCAN_BATCH_MAX_SIZE_IN_BYTES.get(scan_type, consts.DEFAULT_SCAN_BATCH_MAX_SIZE_IN_BYTES) + + +def _get_max_batch_files_count(_: str) -> int: + logger.debug( + 'You can customize the batch files count by setting the environment variable "%s"', + consts.SCAN_BATCH_MAX_FILES_COUNT_ENV_VAR_NAME, + ) + + custom_files_count = os.environ.get(consts.SCAN_BATCH_MAX_FILES_COUNT_ENV_VAR_NAME) + if custom_files_count: + logger.debug('Custom batch files count is set, %s', {'custom_files_count': custom_files_count}) + return int(custom_files_count) + + return consts.DEFAULT_SCAN_BATCH_MAX_FILES_COUNT + + +def split_documents_into_batches( + scan_type: str, + documents: list[Document], +) -> list[list[Document]]: + max_size = _get_max_batch_size(scan_type) + max_files_count = _get_max_batch_files_count(scan_type) + + logger.debug( + 'Splitting documents into batches, %s', + {'document_count': len(documents), 'max_batch_size': max_size, 'max_files_count': max_files_count}, + ) + + batches = [] + + current_size = 0 + current_batch = [] + for document in documents: + document_size = len(document.content.encode('UTF-8')) + + exceeds_max_size = current_size + document_size > max_size + if exceeds_max_size: + logger.debug( + 'Going to create new batch because current batch size exceeds the limit, %s', + { + 'batch_index': len(batches), + 'current_batch_size': current_size + document_size, + 'max_batch_size': max_size, + }, + ) + + exceeds_max_files_count = len(current_batch) >= max_files_count + if exceeds_max_files_count: + logger.debug( + 'Going to create new batch because current batch files count exceeds the limit, %s', + { + 'batch_index': len(batches), + 'current_batch_files_count': len(current_batch), + 'max_batch_files_count': max_files_count, + }, + ) + + if exceeds_max_size or exceeds_max_files_count: + batches.append(current_batch) + + current_batch = [document] + current_size = document_size + else: + current_batch.append(document) + current_size += document_size + + if current_batch: + batches.append(current_batch) + + logger.debug('Documents were split into batches %s', {'batches_count': len(batches)}) + + return batches + + +def _get_threads_count() -> int: + cpu_count = os.cpu_count() or 1 + return min(cpu_count * consts.SCAN_BATCH_SCANS_PER_CPU, consts.SCAN_BATCH_MAX_PARALLEL_SCANS) + + +def run_parallel_batched_scan( + scan_function: Callable[[list[Document]], tuple[str, 'CliError', 'LocalScanResult']], + scan_type: str, + documents: list[Document], + progress_bar: 'BaseProgressBar', + skip_batching: bool = False, +) -> tuple[dict[str, 'CliError'], list['LocalScanResult']]: + # batching is disabled for SCA; requested by Mor + if scan_type == consts.SCA_SCAN_TYPE or skip_batching: + batches = [documents] + else: + batches = split_documents_into_batches(scan_type, documents) + + progress_bar.set_section_length(ScanProgressBarSection.SCAN, len(batches)) # * 3 + # TODO(MarshalX): we should multiply the count of batches in SCAN section because each batch has 3 steps: + # 1. scan creation + # 2. scan completion + # 3. detection creation + # it's not possible yet because not all scan types moved to polling mechanism + # the progress bar could be significant improved (be more dynamic) in the future + + threads_count = _get_threads_count() + local_scan_results: list[LocalScanResult] = [] + cli_errors: dict[str, CliError] = {} + + logger.debug('Running parallel batched scan, %s', {'threads_count': threads_count, 'batches_count': len(batches)}) + + with ThreadPool(processes=threads_count) as pool: + for scan_id, err, result in pool.imap(scan_function, batches): + if result: + local_scan_results.append(result) + if err: + cli_errors[scan_id] = err + + progress_bar.update(ScanProgressBarSection.SCAN) + + return cli_errors, local_scan_results diff --git a/cycode/cli/utils/scan_utils.py b/cycode/cli/utils/scan_utils.py index 8541add1..819a4116 100644 --- a/cycode/cli/utils/scan_utils.py +++ b/cycode/cli/utils/scan_utils.py @@ -1,7 +1,64 @@ -import click +import os +from collections import defaultdict +from typing import TYPE_CHECKING, Optional +from uuid import UUID, uuid4 +import typer -def is_scan_failed(context: click.Context): - did_fail = context.obj.get("did_fail") - issue_detected = context.obj.get("issue_detected") +from cycode.cli import consts +from cycode.cli.cli_types import SeverityOption + +if TYPE_CHECKING: + from cycode.cli.models import LocalScanResult + from cycode.cyclient.models import ScanConfiguration + + +def set_issue_detected(ctx: typer.Context, issue_detected: bool) -> None: + ctx.obj['issue_detected'] = issue_detected + + +def set_issue_detected_by_scan_results(ctx: typer.Context, scan_results: list['LocalScanResult']) -> None: + set_issue_detected(ctx, any(scan_result.issue_detected for scan_result in scan_results)) + + +def is_scan_failed(ctx: typer.Context) -> bool: + did_fail = ctx.obj.get('did_fail') + issue_detected = ctx.obj.get('issue_detected') return did_fail or issue_detected + + +def is_cycodeignore_allowed_by_scan_config(ctx: typer.Context) -> bool: + scan_config: Optional[ScanConfiguration] = ctx.obj.get('scan_config') + return scan_config.is_cycode_ignore_allowed if scan_config else True + + +def should_use_presigned_upload(scan_type: str) -> bool: + return scan_type in consts.PRESIGNED_UPLOAD_SCAN_TYPES + + +def generate_unique_scan_id() -> UUID: + if 'PYTEST_TEST_UNIQUE_ID' in os.environ: + return UUID(os.environ['PYTEST_TEST_UNIQUE_ID']) + + return uuid4() + + +def build_violation_summary(local_scan_results: list['LocalScanResult']) -> str: + """Build violation summary string with severity breakdown and emojis.""" + detections_count = 0 + severity_counts = defaultdict(int) + + for local_scan_result in local_scan_results: + for document_detections in local_scan_result.document_detections: + for detection in document_detections.detections: + if detection.severity: + detections_count += 1 + severity_counts[SeverityOption(detection.severity)] += 1 + + severity_parts = [] + for severity in reversed(SeverityOption): + emoji = SeverityOption.get_member_unicode_emoji(severity) + count = severity_counts[severity] + severity_parts.append(f'{emoji} {severity.upper()} - {count}') + + return f'Cycode found {detections_count} violations: {" | ".join(severity_parts)}' diff --git a/cycode/cli/utils/shell_executor.py b/cycode/cli/utils/shell_executor.py index ca2e8caa..b39d2a0b 100644 --- a/cycode/cli/utils/shell_executor.py +++ b/cycode/cli/utils/shell_executor.py @@ -1,33 +1,55 @@ import subprocess -from typing import List, Optional, Union +import time +from typing import Optional, Union import click +import typer -from cycode.cyclient import logger +from cycode.logger import get_logger _SUBPROCESS_DEFAULT_TIMEOUT_SEC = 60 +logger = get_logger('SHELL') + + def shell( - command: Union[str, List[str]], timeout: int = _SUBPROCESS_DEFAULT_TIMEOUT_SEC, execute_in_shell=False + command: Union[str, list[str]], + timeout: int = _SUBPROCESS_DEFAULT_TIMEOUT_SEC, + working_directory: Optional[str] = None, + silent_exc_info: bool = False, ) -> Optional[str]: - logger.debug(f'Executing shell command: {command}') + logger.debug('Executing shell command: %s', command) try: - result = subprocess.run( - command, - timeout=timeout, - shell=execute_in_shell, - check=True, - capture_output=True, + start = time.monotonic() + result = subprocess.run( # noqa: S603 + command, cwd=working_directory, timeout=timeout, check=True, capture_output=True ) + duration_sec = round(time.monotonic() - start, 2) + stdout = result.stdout.decode('UTF-8').strip() + stderr = result.stderr.decode('UTF-8').strip() - return result.stdout.decode('UTF-8').strip() + logger.debug( + 'Shell command executed successfully, %s', + {'duration_sec': duration_sec, 'stdout': stdout if stdout else '', 'stderr': stderr if stderr else ''}, + ) + + return stdout except subprocess.CalledProcessError as e: - logger.debug(f'Error occurred while running shell command. Exception: {e.stderr}') - except subprocess.TimeoutExpired: - raise click.Abort(f'Command "{command}" timed out') + if not silent_exc_info: + logger.debug('Error occurred while running shell command', exc_info=e) + if e.stdout: + logger.debug('Shell command stdout: %s', e.stdout.decode('UTF-8').strip()) + if e.stderr: + logger.debug('Shell command stderr: %s', e.stderr.decode('UTF-8').strip()) + except subprocess.TimeoutExpired as e: + logger.debug('Command timed out', exc_info=e) + raise typer.Abort(f'Command "{command}" timed out') from e except Exception as e: - raise click.ClickException(f'Unhandled exception: {e}') + if not silent_exc_info: + logger.debug('Unhandled exception occurred while running shell command', exc_info=e) + + raise click.ClickException(f'Unhandled exception: {e}') from e return None diff --git a/cycode/cli/utils/string_utils.py b/cycode/cli/utils/string_utils.py index 0e7d0c23..43931239 100644 --- a/cycode/cli/utils/string_utils.py +++ b/cycode/cli/utils/string_utils.py @@ -1,10 +1,12 @@ import hashlib import math -import re import random +import re import string from sys import getsizeof -from binaryornot.check import is_binary_string + +from cycode.cli.consts import SCA_SHORTCUT_DEPENDENCY_PATHS +from cycode.cli.utils.binary_utils import is_binary_string def obfuscate_text(text: str) -> str: @@ -12,12 +14,12 @@ def obfuscate_text(text: str) -> str: start_reveled_len = math.ceil(match_len / 8) end_reveled_len = match_len - (math.ceil(match_len / 8)) - obfuscated = obfuscate_regex.sub("*", text) + obfuscated = obfuscate_regex.sub('*', text) return f'{text[:start_reveled_len]}{obfuscated[start_reveled_len:end_reveled_len]}{text[end_reveled_len:]}' -obfuscate_regex = re.compile(r"[^+\-\s]") +obfuscate_regex = re.compile(r'[^+\-\s]') def is_binary_content(content: str) -> bool: @@ -27,23 +29,47 @@ def is_binary_content(content: str) -> bool: return is_binary_string(chunk_bytes) -def get_content_size(content: str): +def get_content_size(content: str) -> int: return getsizeof(content) -def convert_string_to_bytes(content: str): - return bytes(content, 'utf-8') +def convert_string_to_bytes(content: str) -> bytes: + return bytes(content, 'UTF-8') -def hash_string_to_sha256(content: str): +def hash_string_to_sha256(content: str) -> str: return hashlib.sha256(content.encode()).hexdigest() -def generate_random_string(string_len: int): +def generate_random_string(string_len: int) -> str: # letters, digits, and symbols characters = string.ascii_letters + string.digits + string.punctuation - return ''.join(random.choice(characters) for _ in range(string_len)) + return ''.join(random.choice(characters) for _ in range(string_len)) # noqa: S311 def get_position_in_line(text: str, position: int) -> int: return position - text.rfind('\n', 0, position) - 1 + + +def shortcut_dependency_paths(dependency_paths_list: str) -> str: + separate_dependency_paths_list = dependency_paths_list.split(',') + result = '' + for dependency_paths in separate_dependency_paths_list: + dependency_paths = dependency_paths.strip().rstrip() + dependencies = dependency_paths.split(' -> ') + if len(dependencies) <= SCA_SHORTCUT_DEPENDENCY_PATHS: + result += dependency_paths + else: + result += f'{dependencies[0]} -> ... -> {dependencies[-1]}' + result += '\n' + + return result.rstrip().rstrip(',') + + +def sanitize_text_for_encoding(text: str) -> str: + """Sanitize text by replacing surrogate characters and invalid UTF-8 sequences. + + This prevents encoding errors when Rich tries to display the content, especially on Windows. + Surrogate characters (U+D800 to U+DFFF) cannot be encoded to UTF-8 and will cause errors. + """ + return text.encode('utf-8', errors='replace').decode('utf-8') diff --git a/cycode/cli/utils/task_timer.py b/cycode/cli/utils/task_timer.py index dadc0aea..4b5e903e 100644 --- a/cycode/cli/utils/task_timer.py +++ b/cycode/cli/utils/task_timer.py @@ -1,51 +1,48 @@ -from threading import Thread, Event from _thread import interrupt_main -from typing import Optional, Callable, List, Dict, Type +from threading import Event, Thread from types import TracebackType +from typing import Callable, Optional class FunctionContext: - - def __init__(self, function: Callable, args: List = None, kwargs: Dict = None): + def __init__(self, function: Callable, args: Optional[list] = None, kwargs: Optional[dict] = None) -> None: self.function = function self.args = args or [] self.kwargs = kwargs or {} class TimerThread(Thread): - """ - Custom thread class for executing timer in the background + """Custom thread class for executing timer in the background. Members: timeout - the amount of time to count until timeout in seconds quit_function (Mandatory) - function to perform when reaching to timeout """ - def __init__(self, timeout: int, - quit_function: FunctionContext): + + def __init__(self, timeout: int, quit_function: FunctionContext) -> None: Thread.__init__(self) self._timeout = timeout self._quit_function = quit_function self.event = Event() - def run(self): + def run(self) -> None: self._run_quit_function_on_timeout() - def stop(self): + def stop(self) -> None: self.event.set() - def _run_quit_function_on_timeout(self): + def _run_quit_function_on_timeout(self) -> None: self.event.wait(self._timeout) if not self.event.is_set(): self._call_quit_function() self.stop() - def _call_quit_function(self): + def _call_quit_function(self) -> None: self._quit_function.function(*self._quit_function.args, **self._quit_function.kwargs) class TimeoutAfter: - """ - A task wrapper for controlling how much time a task should be run before timing out + """A task wrapper for controlling how much time a task should be run before timing out. Use Example: with TimeoutAfter(5, repeat_function=FunctionContext(x), repeat_interval=2): @@ -56,8 +53,8 @@ class TimeoutAfter: quit_function (Optional) - function to perform when reaching to timeout, the default option is to interrupt main thread """ - def __init__(self, timeout: int, - quit_function: Optional[FunctionContext] = None): + + def __init__(self, timeout: int, quit_function: Optional[FunctionContext] = None) -> None: self.timeout = timeout self._quit_function = quit_function or FunctionContext(function=self.timeout_function) self.timer = TimerThread(timeout, quit_function=self._quit_function) @@ -66,15 +63,16 @@ def __enter__(self) -> None: if self.timeout: self.timer.start() - def __exit__(self, exc_type: Optional[Type[BaseException]], exc_val: Optional[BaseException], - exc_tb: Optional[TracebackType]) -> None: + def __exit__( + self, exc_type: Optional[type[BaseException]], exc_val: Optional[BaseException], exc_tb: Optional[TracebackType] + ) -> None: if self.timeout: self.timer.stop() # catch the exception of interrupt_main before exiting # the with statement and throw timeout error instead - if exc_type == KeyboardInterrupt: - raise TimeoutError(f"Task timed out after {self.timeout} seconds") + if exc_type is KeyboardInterrupt: + raise TimeoutError(f'Task timed out after {self.timeout} seconds') - def timeout_function(self): + def timeout_function(self) -> None: interrupt_main() diff --git a/cycode/cli/utils/url_utils.py b/cycode/cli/utils/url_utils.py new file mode 100644 index 00000000..91e50f77 --- /dev/null +++ b/cycode/cli/utils/url_utils.py @@ -0,0 +1,64 @@ +from typing import Optional +from urllib.parse import urlparse, urlunparse + +from cycode.logger import get_logger + +logger = get_logger('URL Utils') + + +def sanitize_repository_url(url: Optional[str]) -> Optional[str]: + """Remove credentials (username, password, tokens) from repository URL. + + This function sanitizes repository URLs to prevent sending PAT tokens or other + credentials to the API. It handles both HTTP/HTTPS URLs with embedded credentials + and SSH URLs (which are returned as-is since they don't contain credentials in the URL). + + Args: + url: Repository URL that may contain credentials (e.g., https://token@github.com/user/repo.git) + + Returns: + Sanitized URL without credentials (e.g., https://github.com/user/repo.git), or None if input is None + + Examples: + >>> sanitize_repository_url('https://token@github.com/user/repo.git') + 'https://github.com/user/repo.git' + >>> sanitize_repository_url('https://user:token@github.com/user/repo.git') + 'https://github.com/user/repo.git' + >>> sanitize_repository_url('git@github.com:user/repo.git') + 'git@github.com:user/repo.git' + >>> sanitize_repository_url(None) + None + """ + if not url: + return url + + # Handle SSH URLs - no credentials to remove + # ssh:// URLs have the format ssh://git@host/path + if url.startswith('ssh://'): + return url + # git@host:path format (scp-style) + if '@' in url and '://' not in url and url.startswith('git@'): + return url + + try: + parsed = urlparse(url) + # Remove username and password from netloc + # Reconstruct URL without credentials + sanitized_netloc = parsed.hostname + if parsed.port: + sanitized_netloc = f'{sanitized_netloc}:{parsed.port}' + + return urlunparse( + ( + parsed.scheme, + sanitized_netloc, + parsed.path, + parsed.params, + parsed.query, + parsed.fragment, + ) + ) + except Exception as e: + logger.debug('Failed to sanitize repository URL, returning original, %s', {'url': url, 'error': str(e)}) + # If parsing fails, return original URL to avoid breaking functionality + return url diff --git a/cycode/cli/utils/version_checker.py b/cycode/cli/utils/version_checker.py new file mode 100644 index 00000000..24146989 --- /dev/null +++ b/cycode/cli/utils/version_checker.py @@ -0,0 +1,219 @@ +import os +import re +import time +from pathlib import Path +from typing import Optional + +from cycode.cli.console import console +from cycode.cli.user_settings.configuration_manager import ConfigurationManager +from cycode.cli.utils.path_utils import get_file_content +from cycode.cyclient.cycode_client_base import CycodeClientBase +from cycode.logger import get_logger + +logger = get_logger('Version Checker') + + +def _compare_versions( + current_parts: list[int], + latest_parts: list[int], + current_is_pre: bool, + latest_is_pre: bool, + latest_version: str, +) -> Optional[str]: + """Compare version numbers and determine if an update is needed. + + Implements version comparison logic with special handling for pre-release versions: + - Won't suggest downgrading from stable to pre-release + - Will suggest upgrading from pre-release to stable of the same version + + Args: + current_parts: List of numeric version components for the current version + latest_parts: List of numeric version components for the latest version + current_is_pre: Whether the current version is pre-release + latest_is_pre: Whether the latest version is pre-release + latest_version: The full latest version string + + Returns: + str | None: The latest version string if an update is recommended, + None if no update is needed + + """ + # If current is stable and latest is pre-release, don't suggest update + if not current_is_pre and latest_is_pre: + return None + + # Compare version numbers + for current, latest in zip(current_parts, latest_parts): + if latest > current: + return latest_version + if current > latest: + return None + + # If all numbers are equal, suggest update if current is pre-release and latest is stable + if current_is_pre and not latest_is_pre: + return latest_version + + return None + + +class VersionChecker(CycodeClientBase): + PYPI_API_URL = 'https://pypi.org/pypi' + PYPI_PACKAGE_NAME = 'cycode' + PYPI_REQUEST_TIMEOUT = 1 + + GIT_CHANGELOG_URL_PREFIX = 'https://github.com/cycodehq/cycode-cli/releases/tag/v' + + DAILY = 24 * 60 * 60 # 24 hours in seconds + WEEKLY = DAILY * 7 + + def __init__(self) -> None: + """Initialize the VersionChecker. + + Sets up the version checker with PyPI API URL and configure the cache file location + using the global configuration directory. + """ + super().__init__(self.PYPI_API_URL) + + configuration_manager = ConfigurationManager() + config_dir = configuration_manager.global_config_file_manager.get_config_directory_path() + self.cache_file = Path(config_dir) / '.version_check' + + def get_latest_version(self) -> Optional[str]: + """Fetch the latest version of the package from PyPI. + + Makes an HTTP request to PyPI's JSON API to get the latest version information. + + Returns: + str | None: The latest version string if successful, None if the request fails + or the version information is not available. + + """ + try: + response = self.get( + f'{self.PYPI_PACKAGE_NAME}/json', timeout=self.PYPI_REQUEST_TIMEOUT, hide_response_content_log=True + ) + data = response.json() + return data.get('info', {}).get('version') + except Exception: + return None + + @staticmethod + def _parse_version(version: str) -> tuple[list[int], bool]: + """Parse version string into components and identify if it's a pre-release. + + Extracts numeric version components and determines if the version is a pre-release + by checking for 'dev' in the version string. + + Args: + version: The version string to parse (e.g., '1.2.3' or '1.2.3dev4') + + Returns: + tuple: A tuple containing: + - List[int]: List of numeric version components + - bool: True if this is a pre-release version, False otherwise + + """ + version_parts = [int(x) for x in re.findall(r'\d+', version)] + is_prerelease = 'dev' in version + + return version_parts, is_prerelease + + def _should_check_update(self, is_prerelease: bool) -> bool: + """Determine if an update check should be performed based on the last check time. + + Implements a time-based caching mechanism where update checks are performed: + - Daily for pre-release versions + - Weekly for stable versions + + Args: + is_prerelease: Whether the current version is a pre-release + + Returns: + bool: True if an update check should be performed, False otherwise + + """ + if not os.path.exists(self.cache_file): + return True + + file_content = get_file_content(self.cache_file) + if file_content is None: + return True + + try: + last_check = float(file_content.strip()) + except ValueError: + return True + + duration = self.DAILY if is_prerelease else self.WEEKLY + return time.time() - last_check >= duration + + def _update_last_check(self) -> None: + """Update the timestamp of the last update check. + + Creates the cache directory if it doesn't exist and write the current timestamp + to the cache file. Silently handle any IO errors that might occur during the process. + """ + try: + os.makedirs(os.path.dirname(self.cache_file), exist_ok=True) + with open(self.cache_file, 'w', encoding='UTF-8') as f: + f.write(str(time.time())) + except Exception as e: + logger.debug('Failed to update version check cache file: %s', {'file': self.cache_file}, exc_info=e) + + def check_for_update(self, current_version: str, use_cache: bool = True) -> Optional[str]: + """Check if an update is available for the current version. + + Respects the update check frequency (daily/weekly) based on the version type + + Args: + current_version: The current version string of the CLI + use_cache: If True, use the cached timestamp to determine if an update check is needed + + Returns: + str | None: The latest version string if an update is recommended, + None if no update is needed or if check should be skipped + + """ + current_parts, current_is_pre = self._parse_version(current_version) + + # Check if we should perform the update check based on frequency + if use_cache and not self._should_check_update(current_is_pre): + return None + + latest_version = self.get_latest_version() + if not latest_version: + return None + + # Update the last check timestamp + use_cache and self._update_last_check() + + latest_parts, latest_is_pre = self._parse_version(latest_version) + return _compare_versions(current_parts, latest_parts, current_is_pre, latest_is_pre, latest_version) + + def check_and_notify_update(self, current_version: str, use_cache: bool = True) -> None: + """Check for updates and display a notification if a new version is available. + + Performs the version check and displays a formatted message with update instructions + if a newer version is available. The message includes: + - Current and new version numbers + - Link to the changelog + - Command to perform the update + + Args: + current_version: Current version of the CLI + use_cache: If True, use the cached timestamp to determine if an update check is needed + + """ + latest_version = self.check_for_update(current_version, use_cache) + should_update = bool(latest_version) + if should_update: + update_message = ( + '\nNew release of Cycode CLI is available: ' + f'[red]{current_version}[/] -> [green]{latest_version}[/]\n' + f'Changelog: [bright_blue]{self.GIT_CHANGELOG_URL_PREFIX}{latest_version}[/]\n' + f'To update, run: [green]pip install --upgrade cycode[/]\n' + ) + console.print(update_message) + + +version_checker = VersionChecker() diff --git a/cycode/cli/utils/yaml_utils.py b/cycode/cli/utils/yaml_utils.py index e1732ef8..c92acdc8 100644 --- a/cycode/cli/utils/yaml_utils.py +++ b/cycode/cli/utils/yaml_utils.py @@ -1,28 +1,56 @@ -import yaml -from typing import Dict - - -def read_file(filename: str) -> Dict: - with open(filename, 'r', encoding="utf-8") as file: - return yaml.safe_load(file) +import os +from collections.abc import Hashable +from typing import Any, TextIO +import yaml -def update_file(filename: str, content: Dict): - try: - with open(filename, 'r', encoding="utf-8") as file: - file_content = yaml.safe_load(file) - except FileNotFoundError: - file_content = {} +from cycode.logger import get_logger - with open(filename, 'w', encoding="utf-8") as file: - file_content = _deep_update(file_content, content) - yaml.safe_dump(file_content, file) +logger = get_logger('YAML Utils') -def _deep_update(source, overrides): +def _deep_update(source: dict[Hashable, Any], overrides: dict[Hashable, Any]) -> dict[Hashable, Any]: for key, value in overrides.items(): if isinstance(value, dict) and value: source[key] = _deep_update(source.get(key, {}), value) else: source[key] = overrides[key] + return source + + +def _yaml_object_safe_load(file: TextIO) -> dict[Hashable, Any]: + # loader.get_single_data could return None + loaded_file = yaml.safe_load(file) + + if not isinstance(loaded_file, dict): + # forbid literals at the top level + logger.debug( + 'YAML file does not contain a dictionary at the top level: %s', + {'filename': file.name, 'actual_type': type(loaded_file)}, + ) + return {} + + return loaded_file + + +def read_yaml_file(filename: str) -> dict[Hashable, Any]: + if not os.access(filename, os.R_OK) or not os.path.exists(filename): + logger.debug('Config file is not accessible or does not exist: %s', {'filename': filename}) + return {} + + with open(filename, encoding='UTF-8') as file: + return _yaml_object_safe_load(file) + + +def write_yaml_file(filename: str, content: dict[Hashable, Any]) -> None: + if not os.access(filename, os.W_OK) and os.path.exists(filename): + logger.warning('No write permission for file. Cannot save config, %s', {'filename': filename}) + return + + with open(filename, 'w', encoding='UTF-8') as file: + yaml.safe_dump(content, file) + + +def update_yaml_file(filename: str, content: dict[Hashable, Any]) -> None: + write_yaml_file(filename, _deep_update(read_yaml_file(filename), content)) diff --git a/cycode/cli/zip_file.py b/cycode/cli/zip_file.py deleted file mode 100644 index 3a23f9a2..00000000 --- a/cycode/cli/zip_file.py +++ /dev/null @@ -1,33 +0,0 @@ -import os.path -from zipfile import ZipFile, ZIP_DEFLATED -from io import BytesIO - - -class InMemoryZip(object): - def __init__(self): - # Create the in-memory file-like object - self.in_memory_zip = BytesIO() - self.zip = ZipFile(self.in_memory_zip, "a", ZIP_DEFLATED, False) - - def append(self, filename, unique_id, content): - # Write the file to the in-memory zip - if unique_id: - filename = concat_unique_id(filename, unique_id) - - self.zip.writestr(filename, content) - - def close(self): - self.zip.close() - - # to bytes - def read(self) -> bytes: - self.in_memory_zip.seek(0) - return self.in_memory_zip.read() - - -def concat_unique_id(filename: str, unique_id: str) -> str: - if filename.startswith(os.sep): - # remove leading slash to join path correctly - filename = filename[len(os.sep):] - - return os.path.join(unique_id, filename) diff --git a/cycode/config.py b/cycode/config.py new file mode 100644 index 00000000..f4306b31 --- /dev/null +++ b/cycode/config.py @@ -0,0 +1,45 @@ +import logging +import os +from typing import Optional +from urllib.parse import urlparse + +from cycode.cli import consts +from cycode.cyclient import config_dev + +DEFAULT_CONFIGURATION = { + consts.TIMEOUT_ENV_VAR_NAME: 300, + consts.LOGGING_LEVEL_ENV_VAR_NAME: logging.INFO, + config_dev.DEV_MODE_ENV_VAR_NAME: 'false', +} + +configuration = dict(DEFAULT_CONFIGURATION, **os.environ) + + +def get_val_as_string(key: str) -> str: + return configuration.get(key) + + +def get_val_as_bool(key: str, default: bool = False) -> bool: + if key not in configuration: + return default + + return configuration[key].lower() in {'true', '1', 'yes', 'y', 'on', 'enabled'} + + +def get_val_as_int(key: str) -> Optional[int]: + val = configuration.get(key) + if not val: + return None + + try: + return int(val) + except ValueError: + return None + + +def is_valid_url(url: str) -> bool: + try: + parsed_url = urlparse(url) + return all([parsed_url.scheme, parsed_url.netloc]) + except ValueError: + return False diff --git a/cycode/cyclient/__init__.py b/cycode/cyclient/__init__.py index f33d4b71..e69de29b 100644 --- a/cycode/cyclient/__init__.py +++ b/cycode/cyclient/__init__.py @@ -1,6 +0,0 @@ -from .config import logger - - -__all__ = [ - "logger", -] diff --git a/cycode/cyclient/ai_security_manager_client.py b/cycode/cyclient/ai_security_manager_client.py new file mode 100644 index 00000000..35c1d8c9 --- /dev/null +++ b/cycode/cyclient/ai_security_manager_client.py @@ -0,0 +1,90 @@ +"""Client for AI Security Manager service.""" + +from typing import TYPE_CHECKING, Optional + +from cycode.cli.exceptions.custom_exceptions import HttpUnauthorizedError +from cycode.cyclient.cycode_client_base import CycodeClientBase +from cycode.cyclient.logger import logger + +if TYPE_CHECKING: + from cycode.cli.apps.ai_guardrails.scan.payload import AIHookPayload + from cycode.cli.apps.ai_guardrails.scan.types import AiHookEventType, AIHookOutcome, BlockReason + from cycode.cyclient.ai_security_manager_service_config import AISecurityManagerServiceConfigBase + + +class AISecurityManagerClient: + """Client for interacting with AI Security Manager service.""" + + _CONVERSATIONS_PATH = 'v4/ai-security/interactions/conversations' + _EVENTS_PATH = 'v4/ai-security/interactions/events' + + def __init__(self, client: CycodeClientBase, service_config: 'AISecurityManagerServiceConfigBase') -> None: + self.client = client + self.service_config = service_config + + def _build_endpoint_path(self, path: str) -> str: + """Build the full endpoint path including service name/port.""" + service_name = self.service_config.get_service_name() + if service_name: + return f'{service_name}/{path}' + return path + + def create_conversation(self, payload: 'AIHookPayload') -> Optional[str]: + """Creates an AI conversation from hook payload.""" + conversation_id = payload.conversation_id + if not conversation_id: + return None + + body = { + 'id': conversation_id, + 'ide_user_email': payload.ide_user_email, + 'model': payload.model, + 'ide_provider': payload.ide_provider, + 'ide_version': payload.ide_version, + } + + try: + self.client.post(self._build_endpoint_path(self._CONVERSATIONS_PATH), body=body) + except HttpUnauthorizedError: + # Authentication error - re-raise so prompt_command can catch it + raise + except Exception as e: + logger.debug('Failed to create conversation', exc_info=e) + # Don't fail the hook if tracking fails (non-auth errors) + + return conversation_id + + def create_event( + self, + payload: 'AIHookPayload', + event_type: 'AiHookEventType', + outcome: 'AIHookOutcome', + scan_id: Optional[str] = None, + block_reason: Optional['BlockReason'] = None, + error_message: Optional[str] = None, + file_path: Optional[str] = None, + ) -> None: + """Create an AI hook event from hook payload.""" + conversation_id = payload.conversation_id + if not conversation_id: + logger.debug('No conversation ID available, skipping event creation') + return + + body = { + 'conversation_id': conversation_id, + 'event_type': event_type, + 'outcome': outcome, + 'generation_id': payload.generation_id, + 'block_reason': block_reason, + 'cli_scan_id': scan_id, + 'mcp_server_name': payload.mcp_server_name, + 'mcp_tool_name': payload.mcp_tool_name, + 'error_message': error_message, + 'file_path': file_path, + } + + try: + self.client.post(self._build_endpoint_path(self._EVENTS_PATH), body=body) + except Exception as e: + logger.debug('Failed to create AI hook event', exc_info=e) + # Don't fail the hook if tracking fails diff --git a/cycode/cyclient/ai_security_manager_service_config.py b/cycode/cyclient/ai_security_manager_service_config.py new file mode 100644 index 00000000..60d7f2dd --- /dev/null +++ b/cycode/cyclient/ai_security_manager_service_config.py @@ -0,0 +1,27 @@ +"""Service configuration for AI Security Manager.""" + + +class AISecurityManagerServiceConfigBase: + """Base class for AI Security Manager service configuration.""" + + def get_service_name(self) -> str: + """Get the service name or port for URL construction. + + In dev mode, returns the port number. + In production, returns the service name. + """ + raise NotImplementedError + + +class DevAISecurityManagerServiceConfig(AISecurityManagerServiceConfigBase): + """Dev configuration for AI Security Manager.""" + + def get_service_name(self) -> str: + return '5163/api' + + +class DefaultAISecurityManagerServiceConfig(AISecurityManagerServiceConfigBase): + """Production configuration for AI Security Manager.""" + + def get_service_name(self) -> str: + return '' diff --git a/cycode/cyclient/auth_client.py b/cycode/cyclient/auth_client.py index 379e9bf1..1df7ad9b 100644 --- a/cycode/cyclient/auth_client.py +++ b/cycode/cyclient/auth_client.py @@ -1,18 +1,23 @@ from typing import Optional -from requests import Response +from requests import Request, Response -from .cycode_client import CycodeClient -from . import models -from cycode.cli.exceptions.custom_exceptions import NetworkError, HttpUnauthorizedError +from cycode.cli.exceptions.custom_exceptions import HttpUnauthorizedError, RequestHttpError +from cycode.cyclient import config, models +from cycode.cyclient.cycode_client import CycodeClient class AuthClient: AUTH_CONTROLLER_PATH = 'api/v1/device-auth' - def __init__(self): + def __init__(self) -> None: self.cycode_client = CycodeClient() + @staticmethod + def build_login_url(code_challenge: str, session_id: str) -> str: + query_params = {'source': 'cycode_cli', 'code_challenge': code_challenge, 'session_id': session_id} + return Request(url=f'{config.cycode_app_url}/account/sign-in', params=query_params).prepare().url + def start_session(self, code_challenge: str) -> models.AuthenticationSession: path = f'{self.AUTH_CONTROLLER_PATH}/start' body = {'code_challenge': code_challenge} @@ -23,9 +28,9 @@ def get_api_token(self, session_id: str, code_verifier: str) -> Optional[models. path = f'{self.AUTH_CONTROLLER_PATH}/token' body = {'session_id': session_id, 'code_verifier': code_verifier} try: - response = self.cycode_client.post(url_path=path, body=body) + response = self.cycode_client.post(url_path=path, body=body, hide_response_content_log=True) return self.parse_api_token_polling_response(response) - except (NetworkError, HttpUnauthorizedError) as e: + except (RequestHttpError, HttpUnauthorizedError) as e: return self.parse_api_token_polling_response(e.response) except Exception: return None diff --git a/cycode/cyclient/base_token_auth_client.py b/cycode/cyclient/base_token_auth_client.py new file mode 100644 index 00000000..3f164836 --- /dev/null +++ b/cycode/cyclient/base_token_auth_client.py @@ -0,0 +1,100 @@ +from abc import ABC, abstractmethod +from threading import Lock +from typing import Any, Optional + +import arrow +from requests import Response + +from cycode.cli.user_settings.credentials_manager import CredentialsManager +from cycode.cli.user_settings.jwt_creator import JwtCreator +from cycode.cyclient.cycode_client import CycodeClient + +_NGINX_PLAIN_ERRORS = [ + b'Invalid JWT Token', + b'JWT Token Needed', + b'JWT Token validation failed', +] + + +class BaseTokenAuthClient(CycodeClient, ABC): + """Base client for token-based authentication flows with cached JWTs.""" + + def __init__(self, client_id: str) -> None: + super().__init__() + self.client_id = client_id + + self._credentials_manager = CredentialsManager() + # load cached access token + access_token, expires_in, creator = self._credentials_manager.get_access_token() + + self._access_token = self._expires_in = None + expected_creator = self._create_jwt_creator() + if creator == expected_creator: + # we must be sure that cached access token is created using the same client id and client secret. + # because client id and client secret could be passed via command, via env vars or via config file. + # we must not use cached access token if client id or client secret was changed. + self._access_token = access_token + self._expires_in = arrow.get(expires_in) if expires_in else None + + self._lock = Lock() + + def get_access_token(self) -> str: + with self._lock: + self.refresh_access_token_if_needed() + return self._access_token + + def invalidate_access_token(self, in_storage: bool = False) -> None: + self._access_token = None + self._expires_in = None + + if in_storage: + self._credentials_manager.update_access_token(None, None, None) + + def refresh_access_token_if_needed(self) -> None: + if self._access_token is None or self._expires_in is None or arrow.utcnow() >= self._expires_in: + self.refresh_access_token() + + def refresh_access_token(self) -> None: + auth_response = self._request_new_access_token() + self._access_token = auth_response['token'] + + self._expires_in = arrow.utcnow().shift(seconds=auth_response['expires_in'] * 0.8) + + jwt_creator = self._create_jwt_creator() + self._credentials_manager.update_access_token(self._access_token, self._expires_in.timestamp(), jwt_creator) + + def get_request_headers(self, additional_headers: Optional[dict] = None, without_auth: bool = False) -> dict: + headers = super().get_request_headers(additional_headers=additional_headers) + + if not without_auth: + headers = self._add_auth_header(headers) + + return headers + + def _add_auth_header(self, headers: dict) -> dict: + headers['Authorization'] = f'Bearer {self.get_access_token()}' + return headers + + def _execute( + self, + *args, + **kwargs, + ) -> Response: + response = super()._execute(*args, **kwargs) + + # backend returns 200 and plain text. no way to catch it with .raise_for_status() + nginx_error_response = any(response.content.startswith(plain_error) for plain_error in _NGINX_PLAIN_ERRORS) + if response.status_code == 200 and nginx_error_response: + # if cached token is invalid, try to refresh it and retry the request + self.refresh_access_token() + response = super()._execute(*args, **kwargs) + + return response + + @abstractmethod + def _create_jwt_creator(self) -> JwtCreator: + """Create a JwtCreator instance for the current credential type.""" + + @abstractmethod + def _request_new_access_token(self) -> dict[str, Any]: + """Return the authentication payload with token and expires_in.""" diff --git a/cycode/cyclient/client_creator.py b/cycode/cyclient/client_creator.py new file mode 100644 index 00000000..c26795c7 --- /dev/null +++ b/cycode/cyclient/client_creator.py @@ -0,0 +1,71 @@ +from typing import Optional + +from cycode.cyclient.ai_security_manager_client import AISecurityManagerClient +from cycode.cyclient.ai_security_manager_service_config import ( + DefaultAISecurityManagerServiceConfig, + DevAISecurityManagerServiceConfig, +) +from cycode.cyclient.config import dev_mode +from cycode.cyclient.config_dev import DEV_CYCODE_API_URL +from cycode.cyclient.cycode_dev_based_client import CycodeDevBasedClient +from cycode.cyclient.cycode_oidc_based_client import CycodeOidcBasedClient +from cycode.cyclient.cycode_token_based_client import CycodeTokenBasedClient +from cycode.cyclient.import_sbom_client import ImportSbomClient +from cycode.cyclient.report_client import ReportClient +from cycode.cyclient.scan_client import ScanClient +from cycode.cyclient.scan_config_base import DefaultScanConfig, DevScanConfig + + +def create_scan_client( + client_id: str, client_secret: Optional[str] = None, hide_response_log: bool = False, id_token: Optional[str] = None +) -> ScanClient: + if dev_mode: + client = CycodeDevBasedClient(DEV_CYCODE_API_URL) + scan_config = DevScanConfig() + else: + if id_token: + client = CycodeOidcBasedClient(client_id, id_token) + else: + client = CycodeTokenBasedClient(client_id, client_secret) + scan_config = DefaultScanConfig() + + return ScanClient(client, scan_config, hide_response_log) + + +def create_report_client( + client_id: str, client_secret: Optional[str] = None, _: bool = False, id_token: Optional[str] = None +) -> ReportClient: + if dev_mode: + client = CycodeDevBasedClient(DEV_CYCODE_API_URL) + elif id_token: + client = CycodeOidcBasedClient(client_id, id_token) + else: + client = CycodeTokenBasedClient(client_id, client_secret) + return ReportClient(client) + + +def create_import_sbom_client( + client_id: str, client_secret: Optional[str] = None, _: bool = False, id_token: Optional[str] = None +) -> ImportSbomClient: + if dev_mode: + client = CycodeDevBasedClient(DEV_CYCODE_API_URL) + elif id_token: + client = CycodeOidcBasedClient(client_id, id_token) + else: + client = CycodeTokenBasedClient(client_id, client_secret) + return ImportSbomClient(client) + + +def create_ai_security_manager_client( + client_id: str, client_secret: Optional[str] = None, _: bool = False, id_token: Optional[str] = None +) -> AISecurityManagerClient: + if dev_mode: + client = CycodeDevBasedClient(DEV_CYCODE_API_URL) + service_config = DevAISecurityManagerServiceConfig() + else: + if id_token: + client = CycodeOidcBasedClient(client_id, id_token) + else: + client = CycodeTokenBasedClient(client_id, client_secret) + service_config = DefaultAISecurityManagerServiceConfig() + return AISecurityManagerClient(client, service_config) diff --git a/cycode/cyclient/config.py b/cycode/cyclient/config.py index 5946265c..ec21efb4 100644 --- a/cycode/cyclient/config.py +++ b/cycode/cyclient/config.py @@ -1,79 +1,36 @@ -import logging -import os -import sys -from urllib.parse import urlparse - -from cycode.cli.consts import * +from cycode.cli import consts from cycode.cli.user_settings.configuration_manager import ConfigurationManager -# set io encoding (for windows) -from .config_dev import DEV_MODE_ENV_VAR_NAME, DEV_TENANT_ID_ENV_VAR_NAME - -sys.stdout.reconfigure(encoding='utf-8') -sys.stderr.reconfigure(encoding='utf-8') - -# logs -logging.basicConfig( - stream=sys.stdout, - level=logging.DEBUG, - format="%(asctime)s [%(name)s] %(levelname)s: %(message)s", - datefmt="%Y-%m-%d %H:%M:%S" -) -logging.getLogger("urllib3").setLevel(logging.WARNING) -logging.getLogger("werkzeug").setLevel(logging.WARNING) -logging.getLogger("schedule").setLevel(logging.WARNING) -logging.getLogger("kubernetes").setLevel(logging.WARNING) -logging.getLogger("binaryornot").setLevel(logging.WARNING) -logging.getLogger("chardet").setLevel(logging.WARNING) -logging.getLogger("git.cmd").setLevel(logging.WARNING) -logging.getLogger("git.util").setLevel(logging.WARNING) - -# configs -DEFAULT_CONFIGURATION = { - TIMEOUT_ENV_VAR_NAME: 300, - LOGGING_LEVEL_ENV_VAR_NAME: logging.INFO, - DEV_MODE_ENV_VAR_NAME: 'False', - BATCH_SIZE_ENV_VAR_NAME: 20 -} - -configuration = dict(DEFAULT_CONFIGURATION, **os.environ) - - -def get_logger(logger_name=None): - logger = logging.getLogger(logger_name) - level = _get_val_as_string(LOGGING_LEVEL_ENV_VAR_NAME) - level = level if level in logging._nameToLevel.keys() else int(level) - logger.setLevel(level) +from cycode.config import get_val_as_bool, get_val_as_int, get_val_as_string, is_valid_url +from cycode.cyclient import config_dev +from cycode.cyclient.logger import logger - return logger - - -def _get_val_as_string(key): - return configuration.get(key) +configuration_manager = ConfigurationManager() +cycode_api_url = configuration_manager.get_cycode_api_url() +if not is_valid_url(cycode_api_url): + logger.warning( + 'Invalid Cycode API URL: %s, using default value (%s)', cycode_api_url, consts.DEFAULT_CYCODE_API_URL + ) + cycode_api_url = consts.DEFAULT_CYCODE_API_URL -def _get_val_as_bool(key, default=''): - val = configuration.get(key, default) - return val.lower() in ('true', '1') +cycode_app_url = configuration_manager.get_cycode_app_url() +if not is_valid_url(cycode_app_url): + logger.warning( + 'Invalid Cycode APP URL: %s, using default value (%s)', cycode_app_url, consts.DEFAULT_CYCODE_APP_URL + ) + cycode_app_url = consts.DEFAULT_CYCODE_APP_URL -def _get_val_as_int(key): - val = configuration.get(key) - return int(val) if val is not None else None +def _is_on_premise_installation(cycode_domain: str) -> bool: + return not cycode_api_url.endswith(cycode_domain) -logger = get_logger("cycode cli") -configuration_manager = ConfigurationManager() +on_premise_installation = _is_on_premise_installation(consts.DEFAULT_CYCODE_DOMAIN) -cycode_api_url = configuration_manager.get_cycode_api_url() -try: - urlparse(cycode_api_url) -except ValueError as e: - logger.warning(f'Invalid cycode api url: {cycode_api_url}, using default value', e) - cycode_api_url = DEFAULT_CYCODE_API_URL +timeout = get_val_as_int(consts.CYCODE_CLI_REQUEST_TIMEOUT_ENV_VAR_NAME) +if not timeout: + timeout = get_val_as_int(consts.TIMEOUT_ENV_VAR_NAME) -timeout = _get_val_as_int(CYCODE_CLI_REQUEST_TIMEOUT_ENV_VAR_NAME) or _get_val_as_int(TIMEOUT_ENV_VAR_NAME) -dev_mode = _get_val_as_bool(DEV_MODE_ENV_VAR_NAME) -dev_tenant_id = _get_val_as_string(DEV_TENANT_ID_ENV_VAR_NAME) -batch_size = _get_val_as_int(BATCH_SIZE_ENV_VAR_NAME) -verbose = _get_val_as_bool(VERBOSE_ENV_VAR_NAME) +dev_mode = get_val_as_bool(config_dev.DEV_MODE_ENV_VAR_NAME) +dev_tenant_id = get_val_as_string(config_dev.DEV_TENANT_ID_ENV_VAR_NAME) diff --git a/cycode/cyclient/config.yaml b/cycode/cyclient/config.yaml deleted file mode 100644 index 7b8c1bdc..00000000 --- a/cycode/cyclient/config.yaml +++ /dev/null @@ -1,5 +0,0 @@ -cycode: - api: - base_url: http://api.cycode.com - time_out: 30 -# base_url: http://localhost:5048 #local configuration diff --git a/cycode/cyclient/config_dev.py b/cycode/cyclient/config_dev.py index 1033c7e7..6345c8f8 100644 --- a/cycode/cyclient/config_dev.py +++ b/cycode/cyclient/config_dev.py @@ -1,3 +1,3 @@ -DEV_CYCODE_API_URL = "http://localhost" -DEV_MODE_ENV_VAR_NAME = "DEV_MODE" -DEV_TENANT_ID_ENV_VAR_NAME = "DEV_TENANT_ID" \ No newline at end of file +DEV_CYCODE_API_URL = 'http://localhost' +DEV_MODE_ENV_VAR_NAME = 'DEV_MODE' +DEV_TENANT_ID_ENV_VAR_NAME = 'DEV_TENANT_ID' diff --git a/cycode/cyclient/cycode_client.py b/cycode/cyclient/cycode_client.py index ed3781f7..eded92da 100644 --- a/cycode/cyclient/cycode_client.py +++ b/cycode/cyclient/cycode_client.py @@ -1,8 +1,8 @@ -from . import config -from .cycode_client_base import CycodeClientBase +from cycode.cyclient import config +from cycode.cyclient.cycode_client_base import CycodeClientBase class CycodeClient(CycodeClientBase): - def __init__(self): + def __init__(self) -> None: super().__init__(config.cycode_api_url) self.timeout = config.timeout diff --git a/cycode/cyclient/cycode_client_base.py b/cycode/cyclient/cycode_client_base.py index 1259e250..4b2e2698 100644 --- a/cycode/cyclient/cycode_client_base.py +++ b/cycode/cyclient/cycode_client_base.py @@ -1,35 +1,102 @@ +import os import platform -from typing import Dict +import ssl +from typing import TYPE_CHECKING, Callable, ClassVar, Optional -from requests import Response, request, exceptions +import requests +from requests import Response, exceptions +from requests.adapters import HTTPAdapter +from tenacity import retry, retry_if_exception, stop_after_attempt, wait_random_exponential -from cycode import __version__ -from . import config -from ..cli.exceptions.custom_exceptions import NetworkError, HttpUnauthorizedError -from ..cli.user_settings.configuration_manager import ConfigurationManager +from cycode.cli.exceptions.custom_exceptions import ( + HttpUnauthorizedError, + RequestConnectionError, + RequestError, + RequestHttpError, + RequestSslError, + RequestTimeoutError, +) +from cycode.cyclient import config +from cycode.cyclient.headers import get_cli_user_agent, get_correlation_id +from cycode.cyclient.logger import logger +if TYPE_CHECKING: + from tenacity import RetryCallState -def get_cli_user_agent() -> str: - """Return base User-Agent of CLI. - Example: CycodeCLI/0.2.3 (OS: Darwin; Arch: arm64; Python: 3.8.16; InstallID: *uuid4*) - """ - app_name = 'CycodeCLI' - version = __version__ +class SystemStorageSslContext(HTTPAdapter): + def init_poolmanager(self, *args, **kwargs) -> None: + default_context = ssl.create_default_context() + default_context.load_default_certs() + kwargs['ssl_context'] = default_context + return super().init_poolmanager(*args, **kwargs) - os = platform.system() - arch = platform.machine() - python_version = platform.python_version() + def cert_verify(self, *args, **kwargs) -> None: + super().cert_verify(*args, **kwargs) + conn = kwargs['conn'] if 'conn' in kwargs else args[0] + conn.ca_certs = None - install_id = ConfigurationManager().get_or_create_installation_id() - return f'{app_name}/{version} (OS: {os}; Arch: {arch}; Python: {python_version}; InstallID: {install_id})' +def _get_request_function() -> Callable: + if os.environ.get('REQUESTS_CA_BUNDLE') or os.environ.get('CURL_CA_BUNDLE'): + return requests.request + + if platform.system() != 'Windows': + return requests.request + + session = requests.Session() + session.mount('https://', SystemStorageSslContext()) + return session.request + + +_REQUEST_ERRORS_TO_RETRY = ( + RequestTimeoutError, + RequestConnectionError, + exceptions.ChunkedEncodingError, + exceptions.ContentDecodingError, +) +_RETRY_MAX_ATTEMPTS = 3 +_RETRY_STOP_STRATEGY = stop_after_attempt(_RETRY_MAX_ATTEMPTS) +_RETRY_WAIT_STRATEGY = wait_random_exponential(multiplier=1, min=2, max=10) + + +def _retry_before_sleep(retry_state: 'RetryCallState') -> None: + exception_name = 'None' + if retry_state.outcome.failed: + exception = retry_state.outcome.exception() + exception_name = f'{exception.__class__.__name__}' + + logger.debug( + 'Retrying request after error: %s. Attempt %s of %s. Upcoming sleep: %s', + exception_name, + retry_state.attempt_number, + _RETRY_MAX_ATTEMPTS, + retry_state.upcoming_sleep, + ) + + +def _should_retry_exception(exception: BaseException) -> bool: + if 'PYTEST_CURRENT_TEST' in os.environ: + # We are running under pytest, don't retry + return False + + # Don't retry client errors (400, 401, etc.) + if isinstance(exception, RequestHttpError): + return not exception.status_code < 500 + + is_request_error = isinstance(exception, _REQUEST_ERRORS_TO_RETRY) + is_server_error = isinstance(exception, RequestHttpError) and exception.status_code >= 500 + + return is_request_error or is_server_error class CycodeClientBase: - MANDATORY_HEADERS: Dict[str, str] = {'User-Agent': get_cli_user_agent()} + MANDATORY_HEADERS: ClassVar[dict[str, str]] = { + 'User-Agent': get_cli_user_agent(), + 'X-Correlation-Id': get_correlation_id(), + } - def __init__(self, api_url: str): + def __init__(self, api_url: str) -> None: self.timeout = config.timeout self.api_url = api_url @@ -41,44 +108,51 @@ def reset_user_agent() -> None: def enrich_user_agent(user_agent_suffix: str) -> None: CycodeClientBase.MANDATORY_HEADERS['User-Agent'] += f' {user_agent_suffix}' - def post( - self, - url_path: str, - body: dict = None, - headers: dict = None, - **kwargs - ) -> Response: + def post(self, url_path: str, body: Optional[dict] = None, headers: Optional[dict] = None, **kwargs) -> Response: return self._execute(method='post', endpoint=url_path, json=body, headers=headers, **kwargs) - def put( - self, - url_path: str, - body: dict = None, - headers: dict = None, - **kwargs - ) -> Response: + def put(self, url_path: str, body: Optional[dict] = None, headers: Optional[dict] = None, **kwargs) -> Response: return self._execute(method='put', endpoint=url_path, json=body, headers=headers, **kwargs) - def get( - self, - url_path: str, - headers: dict = None, - **kwargs - ) -> Response: + def get(self, url_path: str, headers: Optional[dict] = None, **kwargs) -> Response: return self._execute(method='get', endpoint=url_path, headers=headers, **kwargs) + @retry( + retry=retry_if_exception(_should_retry_exception), + stop=_RETRY_STOP_STRATEGY, + wait=_RETRY_WAIT_STRATEGY, + reraise=True, + before_sleep=_retry_before_sleep, + ) def _execute( - self, - method: str, - endpoint: str, - headers: dict = None, - **kwargs + self, + method: str, + endpoint: str, + headers: Optional[dict] = None, + without_auth: bool = False, + hide_response_content_log: bool = False, + **kwargs, ) -> Response: url = self.build_full_url(self.api_url, endpoint) + logger.debug( + 'Executing request, %s', + {'method': method.upper(), 'url': url}, + ) + + timeout = self.timeout + if 'timeout' in kwargs: + timeout = kwargs['timeout'] + del kwargs['timeout'] try: - response = request( - method=method, url=url, timeout=self.timeout, headers=self.get_request_headers(headers), **kwargs + headers = self.get_request_headers(headers, without_auth=without_auth) + request = _get_request_function() + response = request(method=method, url=url, timeout=timeout, headers=headers, **kwargs) + + content = 'HIDDEN' if hide_response_content_log else response.text + logger.debug( + 'Receiving response, %s', + {'status_code': response.status_code, 'url': url, 'content': content}, ) response.raise_for_status() @@ -86,7 +160,7 @@ def _execute( except Exception as e: self._handle_exception(e) - def get_request_headers(self, additional_headers: dict = None) -> dict: + def get_request_headers(self, additional_headers: Optional[dict] = None, **kwargs) -> dict[str, str]: if additional_headers is None: return self.MANDATORY_HEADERS.copy() return {**self.MANDATORY_HEADERS, **additional_headers} @@ -94,19 +168,21 @@ def get_request_headers(self, additional_headers: dict = None) -> dict: def build_full_url(self, url: str, endpoint: str) -> str: return f'{url}/{endpoint}' - def _handle_exception(self, e: Exception): + def _handle_exception(self, e: Exception) -> None: if isinstance(e, exceptions.Timeout): - raise NetworkError(504, 'Timeout Error', e.response) - elif isinstance(e, exceptions.HTTPError): - self._handle_http_exception(e) - elif isinstance(e, exceptions.ConnectionError): - raise NetworkError(502, 'Connection Error', e.response) - else: - raise e + raise RequestTimeoutError from e + if isinstance(e, exceptions.HTTPError): + raise self._get_http_exception(e) from e + if isinstance(e, exceptions.SSLError): + raise RequestSslError from e + if isinstance(e, exceptions.ConnectionError): + raise RequestConnectionError from e + + raise e @staticmethod - def _handle_http_exception(e: exceptions.HTTPError): + def _get_http_exception(e: exceptions.HTTPError) -> RequestError: if e.response.status_code == 401: - raise HttpUnauthorizedError(e.response.text, e.response) + return HttpUnauthorizedError(e.response.text, e.response) - raise NetworkError(e.response.status_code, e.response.text, e.response) + return RequestHttpError(e.response.status_code, e.response.text, e.response) diff --git a/cycode/cyclient/cycode_dev_based_client.py b/cycode/cyclient/cycode_dev_based_client.py index 651e0db4..d8fe1cab 100644 --- a/cycode/cyclient/cycode_dev_based_client.py +++ b/cycode/cyclient/cycode_dev_based_client.py @@ -1,5 +1,7 @@ -from .config import dev_tenant_id -from .cycode_client_base import CycodeClientBase +from typing import Optional + +from cycode.cyclient.config import dev_tenant_id +from cycode.cyclient.cycode_client_base import CycodeClientBase """ Send requests with api token @@ -7,15 +9,14 @@ class CycodeDevBasedClient(CycodeClientBase): - - def __init__(self, api_url): + def __init__(self, api_url: str) -> None: super().__init__(api_url) - def get_request_headers(self, additional_headers: dict = None): + def get_request_headers(self, additional_headers: Optional[dict] = None, **_) -> dict[str, str]: headers = super().get_request_headers(additional_headers=additional_headers) headers['X-Tenant-Id'] = dev_tenant_id return headers - def build_full_url(self, url, endpoint): - return f"{url}:{endpoint}" + def build_full_url(self, url: str, endpoint: str) -> str: + return f'{url}:{endpoint}' diff --git a/cycode/cyclient/cycode_oidc_based_client.py b/cycode/cyclient/cycode_oidc_based_client.py new file mode 100644 index 00000000..5208cf5d --- /dev/null +++ b/cycode/cyclient/cycode_oidc_based_client.py @@ -0,0 +1,24 @@ +from typing import Any + +from cycode.cli.user_settings.jwt_creator import JwtCreator +from cycode.cyclient.base_token_auth_client import BaseTokenAuthClient + + +class CycodeOidcBasedClient(BaseTokenAuthClient): + """Send requests with JWT obtained via OIDC ID token.""" + + def __init__(self, client_id: str, id_token: str) -> None: + self.id_token = id_token + super().__init__(client_id) + + def _request_new_access_token(self) -> dict[str, Any]: + auth_response = self.post( + url_path='api/v1/auth/oidc/api-token', + body={'client_id': self.client_id, 'id_token': self.id_token}, + without_auth=True, + hide_response_content_log=True, + ) + return auth_response.json() + + def _create_jwt_creator(self) -> JwtCreator: + return JwtCreator.create(self.client_id, self.id_token) diff --git a/cycode/cyclient/cycode_token_based_client.py b/cycode/cyclient/cycode_token_based_client.py index e3da7cc2..bc2dc66e 100644 --- a/cycode/cyclient/cycode_token_based_client.py +++ b/cycode/cyclient/cycode_token_based_client.py @@ -1,52 +1,24 @@ -from threading import Lock +from typing import Any -import arrow +from cycode.cli.user_settings.jwt_creator import JwtCreator +from cycode.cyclient.base_token_auth_client import BaseTokenAuthClient -from .cycode_client import CycodeClient +class CycodeTokenBasedClient(BaseTokenAuthClient): + """Send requests with JWT.""" -class CycodeTokenBasedClient(CycodeClient): - """Send requests with api token""" - - def __init__(self, client_id: str, client_secret: str): - super().__init__() + def __init__(self, client_id: str, client_secret: str) -> None: self.client_secret = client_secret - self.client_id = client_id - - self._api_token = None - self._expires_in = None - - self.lock = Lock() - - @property - def api_token(self) -> str: - # TODO(MarshalX): This property performs HTTP request to refresh the token. This must be the method. - with self.lock: - self.refresh_api_token_if_needed() - return self._api_token - - def refresh_api_token_if_needed(self) -> None: - if self._api_token is None or self._expires_in is None or arrow.utcnow() >= self._expires_in: - self.refresh_api_token() + super().__init__(client_id) - def refresh_api_token(self) -> None: + def _request_new_access_token(self) -> dict[str, Any]: auth_response = self.post( - url_path=f'api/v1/auth/api-token', - body={'clientId': self.client_id, 'secret': self.client_secret} + url_path='api/v1/auth/api-token', + body={'clientId': self.client_id, 'secret': self.client_secret}, + without_auth=True, + hide_response_content_log=True, ) - auth_response_data = auth_response.json() - - self._api_token = auth_response_data['token'] - self._expires_in = arrow.utcnow().shift(seconds=auth_response_data['expires_in'] * 0.8) - - def get_request_headers(self, additional_headers: dict = None) -> dict: - headers = super().get_request_headers(additional_headers=additional_headers) - - if not self.lock.locked(): - headers = self._add_auth_header(headers) - - return headers + return auth_response.json() - def _add_auth_header(self, headers: dict) -> dict: - headers['Authorization'] = f'Bearer {self.api_token}' - return headers + def _create_jwt_creator(self) -> JwtCreator: + return JwtCreator.create(self.client_id, self.client_secret) diff --git a/cycode/cyclient/headers.py b/cycode/cyclient/headers.py new file mode 100644 index 00000000..937f4333 --- /dev/null +++ b/cycode/cyclient/headers.py @@ -0,0 +1,47 @@ +import platform +from typing import Optional +from uuid import uuid4 + +from cycode import __version__ +from cycode.cli import consts +from cycode.cli.user_settings.configuration_manager import ConfigurationManager +from cycode.cyclient.logger import logger + + +def get_cli_user_agent() -> str: + """Return base User-Agent of CLI. + + Example: CycodeCLI/0.2.3 (OS: Darwin; Arch: arm64; Python: 3.8.16; InstallID: *uuid4*) + """ + version = __version__ + + os = platform.system() + arch = platform.machine() + python_version = platform.python_version() + + install_id = ConfigurationManager().get_or_create_installation_id() + + return f'{consts.APP_NAME}/{version} (OS: {os}; Arch: {arch}; Python: {python_version}; InstallID: {install_id})' + + +class _CorrelationId: + _id: Optional[str] = None + + def get_correlation_id(self) -> str: + """Get correlation ID. + + Notes: + Used across all requests to correlate logs and metrics. + It doesn't depend on client instances. + Lifetime is the same as the process. + + """ + if self._id is None: + # example: 16fd2706-8baf-433b-82eb-8c7fada847da + self._id = str(uuid4()) + logger.debug('Correlation ID: %s', self._id) + + return self._id + + +get_correlation_id = _CorrelationId().get_correlation_id diff --git a/cycode/cyclient/import_sbom_client.py b/cycode/cyclient/import_sbom_client.py new file mode 100644 index 00000000..d8107cf5 --- /dev/null +++ b/cycode/cyclient/import_sbom_client.py @@ -0,0 +1,81 @@ +import dataclasses +from pathlib import Path +from typing import Optional + +from requests import Response + +from cycode.cli.cli_types import BusinessImpactOption +from cycode.cli.exceptions.custom_exceptions import RequestHttpError +from cycode.cyclient import models +from cycode.cyclient.cycode_client_base import CycodeClientBase + + +@dataclasses.dataclass +class ImportSbomParameters: + Name: str + Vendor: str + BusinessImpact: BusinessImpactOption + Labels: Optional[list[str]] + Owners: Optional[list[str]] + + def _owners_to_ids(self) -> list[str]: + return [] + + def to_request_form(self) -> dict: + form_data = {} + for field in dataclasses.fields(self): + key = field.name + val = getattr(self, key) + if val is None or len(val) == 0: + continue + if isinstance(val, list): + form_data[f'{key}[]'] = val + else: + form_data[key] = val + return form_data + + +class ImportSbomClient: + IMPORT_SBOM_REQUEST_PATH: str = 'v4/sbom/import' + GET_USER_ID_REQUEST_PATH: str = 'v4/members' + + def __init__(self, client: CycodeClientBase) -> None: + self.client = client + + def request_sbom_import_execution(self, params: ImportSbomParameters, file_path: Path) -> None: + if params.Owners: + owners_ids = self.get_owners_user_ids(params.Owners) + params.Owners = owners_ids + + form_data = params.to_request_form() + + with open(file_path.absolute(), 'rb') as f: + request_args = { + 'url_path': self.IMPORT_SBOM_REQUEST_PATH, + 'data': form_data, + 'files': {'File': f}, + } + + response = self.client.post(**request_args) + + if response.status_code != 201: + raise RequestHttpError(response.status_code, response.text, response) + + def get_owners_user_ids(self, owners: list[str]) -> list[str]: + return [self._get_user_id_by_email(owner) for owner in owners] + + def _get_user_id_by_email(self, email: str) -> str: + request_args = {'url_path': self.GET_USER_ID_REQUEST_PATH, 'params': {'email': email}} + + response = self.client.get(**request_args) + member_details = self.parse_requested_member_details_response(response) + + if not member_details.items: + raise Exception( + f"Failed to find user with email '{email}'. Verify this email is registered to Cycode platform" + ) + return member_details.items.pop(0).external_id + + @staticmethod + def parse_requested_member_details_response(response: Response) -> models.MemberDetails: + return models.RequestedMemberDetailsResultSchema().load(response.json()) diff --git a/cycode/cyclient/logger.py b/cycode/cyclient/logger.py new file mode 100644 index 00000000..b36f036f --- /dev/null +++ b/cycode/cyclient/logger.py @@ -0,0 +1,3 @@ +from cycode.logger import get_logger + +logger = get_logger('CyClient') diff --git a/cycode/cyclient/models.py b/cycode/cyclient/models.py index 9885896e..904fe0ef 100644 --- a/cycode/cyclient/models.py +++ b/cycode/cyclient/models.py @@ -1,12 +1,22 @@ from dataclasses import dataclass -from typing import List, Dict, Optional -from marshmallow import Schema, fields, EXCLUDE, post_load +from typing import Any, Optional + +from marshmallow import EXCLUDE, Schema, fields, post_load class Detection(Schema): - def __init__(self, detection_type_id: str, type: str, message: str, detection_details: dict, - detection_rule_id: str, severity: Optional[str] = None): + def __init__( + self, + detection_type_id: str, + type: str, + message: str, + detection_details: dict, + detection_rule_id: str, + severity: Optional[str] = None, + id: Optional[str] = None, + ) -> None: super().__init__() + self.id = id self.message = message self.type = type self.severity = severity @@ -15,54 +25,58 @@ def __init__(self, detection_type_id: str, type: str, message: str, detection_de self.detection_rule_id = detection_rule_id def __repr__(self) -> str: - return f'type:{self.type}, ' \ - f'severity:{self.severity}, ' \ - f'message:{self.message}, ' \ - f'detection_details:{repr(self.detection_details)}, ' \ - f'detection_rule_id:{self.detection_rule_id}' + return ( + f'type:{self.type}, ' + f'severity:{self.severity}, ' + f'message:{self.message}, ' + f'detection_details:{self.detection_details!r}, ' + f'detection_rule_id:{self.detection_rule_id}' + ) + + @property + def has_alert(self) -> bool: + """Check if the detection has an alert. + + For example, for SCA, it means that the detection is a package vulnerability. + Otherwise, it is a license. + """ + return 'alert' in self.detection_details class DetectionSchema(Schema): class Meta: unknown = EXCLUDE + id = fields.String(load_default=None) message = fields.String() type = fields.String() - severity = fields.String(missing='High') - # TODO(MarshalX): Remove "missing" arg when IaC and Secrets scans will have classifications + severity = fields.String(load_default=None) detection_type_id = fields.String() detection_details = fields.Dict() detection_rule_id = fields.String() @post_load - def build_dto(self, data, **kwargs): + def build_dto(self, data: dict[str, Any], **_) -> Detection: return Detection(**data) class DetectionsPerFile(Schema): - def __init__(self, file_name: str, detections: List[Detection], commit_id: Optional[str] = None): + def __init__(self, file_name: str, detections: list[Detection], commit_id: Optional[str] = None) -> None: super().__init__() self.file_name = file_name self.detections = detections self.commit_id = commit_id -class DetectionsPerFileSchema(Schema): - class Meta: - unknown = EXCLUDE - - file_name = fields.String() - detections = fields.List(fields.Nested(DetectionSchema)) - commit_id = fields.String(allow_none=True) - - @post_load - def build_dto(self, data, **kwargs): - return DetectionsPerFile(**data) - - class ZippedFileScanResult(Schema): - def __init__(self, did_detect: bool, detections_per_file: List[DetectionsPerFile], report_url: Optional[str] = None, - scan_id: str = None, err: str = None): + def __init__( + self, + did_detect: bool, + detections_per_file: list[DetectionsPerFile], + report_url: Optional[str] = None, + scan_id: Optional[str] = None, + err: Optional[str] = None, + ) -> None: super().__init__() self.did_detect = did_detect self.detections_per_file = detections_per_file @@ -71,24 +85,14 @@ def __init__(self, did_detect: bool, detections_per_file: List[DetectionsPerFile self.err = err -class ZippedFileScanResultSchema(Schema): - class Meta: - unknown = EXCLUDE - - did_detect = fields.Boolean() - scan_id = fields.String() - report_url = fields.String(allow_none=True) - detections_per_file = fields.List( - fields.Nested(DetectionsPerFileSchema)) - err = fields.String() - - @post_load - def build_dto(self, data, **kwargs): - return ZippedFileScanResult(**data) - - class ScanResult(Schema): - def __init__(self, did_detect: bool, scan_id: str = None, detections: List[Detection] = None, err: str = None): + def __init__( + self, + did_detect: bool, + scan_id: Optional[str] = None, + detections: Optional[list[Detection]] = None, + err: Optional[str] = None, + ) -> None: super().__init__() self.did_detect = did_detect self.scan_id = scan_id @@ -102,17 +106,36 @@ class Meta: did_detect = fields.Boolean() scan_id = fields.String() - detections = fields.List( - fields.Nested(DetectionSchema), required=False, allow_none=True) + detections = fields.List(fields.Nested(DetectionSchema), required=False, allow_none=True) err = fields.String() @post_load - def build_dto(self, data, **kwargs): + def build_dto(self, data: dict[str, Any], **_) -> 'ScanResult': return ScanResult(**data) +@dataclass +class UploadLinkResponse: + upload_id: str + url: str + presigned_post_fields: dict[str, str] + + +class UploadLinkResponseSchema(Schema): + class Meta: + unknown = EXCLUDE + + upload_id = fields.String() + url = fields.String() + presigned_post_fields = fields.Dict(keys=fields.String(), values=fields.String()) + + @post_load + def build_dto(self, data: dict[str, Any], **_) -> 'UploadLinkResponse': + return UploadLinkResponse(**data) + + class ScanInitializationResponse(Schema): - def __init__(self, scan_id: str = None, err: str = None): + def __init__(self, scan_id: Optional[str] = None, err: Optional[str] = None) -> None: super().__init__() self.scan_id = scan_id self.err = err @@ -126,13 +149,21 @@ class Meta: err = fields.String() @post_load - def build_dto(self, data, **kwargs): + def build_dto(self, data: dict[str, Any], **_) -> 'ScanInitializationResponse': return ScanInitializationResponse(**data) class ScanDetailsResponse(Schema): - def __init__(self, id: str = None, scan_status: str = None, results_count: int = None, metadata: str = None, message: str = None, - scan_update_at: str = None, err: str = None): + def __init__( + self, + id: Optional[str] = None, + scan_status: Optional[str] = None, + results_count: Optional[int] = None, + metadata: Optional[str] = None, + message: Optional[str] = None, + scan_update_at: Optional[str] = None, + err: Optional[str] = None, + ) -> None: super().__init__() self.id = id self.scan_status = scan_status @@ -143,6 +174,19 @@ def __init__(self, id: str = None, scan_status: str = None, results_count: int = self.err = err +@dataclass +class ScanReportUrlResponse: + report_url: str + + +class ScanReportUrlResponseSchema(Schema): + report_url = fields.String() + + @post_load + def build_dto(self, data: dict[str, Any], **_) -> 'ScanReportUrlResponse': + return ScanReportUrlResponse(**data) + + class ScanDetailsResponseSchema(Schema): class Meta: unknown = EXCLUDE @@ -156,12 +200,12 @@ class Meta: err = fields.String() @post_load - def build_dto(self, data, **kwargs): + def build_dto(self, data: dict[str, Any], **_) -> 'ScanDetailsResponse': return ScanDetailsResponse(**data) class K8SResource: - def __init__(self, name: str, resource_type: str, namespace: str, content: Dict): + def __init__(self, name: str, resource_type: str, namespace: str, content: dict) -> None: super().__init__() self.name = name self.type = resource_type @@ -170,23 +214,23 @@ def __init__(self, name: str, resource_type: str, namespace: str, content: Dict) self.internal_metadata = None self.schema = K8SResourceSchema() - def to_json(self): + def to_json(self) -> dict: # FIXME(MarshalX): rename to to_dict? return self.schema.dump(self) class InternalMetadata: - def __init__(self, root_entity_name: str, root_entity_type: str): + def __init__(self, root_entity_name: str, root_entity_type: str) -> None: super().__init__() self.root_entity_name = root_entity_name self.root_entity_type = root_entity_type self.schema = InternalMetadataSchema() - def to_json(self): + def to_json(self) -> dict: # FIXME(MarshalX): rename to to_dict? return self.schema.dump(self) class ResourcesCollection: - def __init__(self, resource_type: str, namespace: str, resources: List[K8SResource], total_count: int): + def __init__(self, resource_type: str, namespace: str, resources: list[K8SResource], total_count: int) -> None: super().__init__() self.type = resource_type self.namespace = namespace @@ -194,7 +238,7 @@ def __init__(self, resource_type: str, namespace: str, resources: List[K8SResour self.total_count = total_count self.schema = ResourcesCollectionSchema() - def to_json(self): + def to_json(self) -> dict: # FIXME(MarshalX): rename to to_dict? return self.schema.dump(self) @@ -219,17 +263,17 @@ class ResourcesCollectionSchema(Schema): class OwnerReference: - def __init__(self, name: str, kind: str): + def __init__(self, name: str, kind: str) -> None: super().__init__() self.name = name self.kind = kind - def __str__(self): - return "Name: {0}, Kind: {1}".format(self.name, self.kind) + def __str__(self) -> str: + return f'Name: {self.name}, Kind: {self.kind}' class AuthenticationSession(Schema): - def __init__(self, session_id: str): + def __init__(self, session_id: str) -> None: super().__init__() self.session_id = session_id @@ -241,12 +285,12 @@ class Meta: session_id = fields.String() @post_load - def build_dto(self, data, **kwargs): + def build_dto(self, data: dict[str, Any], **_) -> 'AuthenticationSession': return AuthenticationSession(**data) class ApiToken(Schema): - def __init__(self, client_id: str, secret: str, description: str): + def __init__(self, client_id: str, secret: str, description: str) -> None: super().__init__() self.client_id = client_id self.secret = secret @@ -262,12 +306,12 @@ class Meta: description = fields.String() @post_load - def build_dto(self, data, **kwargs): + def build_dto(self, data: dict[str, Any], **_) -> 'ApiToken': return ApiToken(**data) class ApiTokenGenerationPollingResponse(Schema): - def __init__(self, status: str, api_token): + def __init__(self, status: str, api_token: 'ApiToken') -> None: super().__init__() self.status = status self.api_token = api_token @@ -281,18 +325,18 @@ class Meta: api_token = fields.Nested(ApiTokenSchema, allow_none=True) @post_load - def build_dto(self, data, **kwargs): + def build_dto(self, data: dict[str, Any], **_) -> 'ApiTokenGenerationPollingResponse': return ApiTokenGenerationPollingResponse(**data) class UserAgentOptionScheme(Schema): app_name = fields.String(required=True) # ex. vscode_extension - app_version = fields.String(required=True) # ex. 0.2.3 + app_version = fields.String(required=True) # ex. 0.2.3 env_name = fields.String(required=True) # ex.: Visual Studio Code - env_version = fields.String(required=True) # ex. 1.78.2 + env_version = fields.String(required=True) # ex. 1.78.2 @post_load - def build_dto(self, data: dict, **_) -> 'UserAgentOption': + def build_dto(self, data: dict[str, Any], **_) -> 'UserAgentOption': return UserAgentOption(**data) @@ -309,8 +353,224 @@ def user_agent_suffix(self) -> str: Example: vscode_extension (AppVersion: 0.1.2; EnvName: vscode; EnvVersion: 1.78.2) """ - return f'{self.app_name} ' \ - f'(' \ - f'AppVersion: {self.app_version}; ' \ - f'EnvName: {self.env_name}; EnvVersion: {self.env_version}' \ - f')' + return ( + f'{self.app_name} ' + f'(' + f'AppVersion: {self.app_version}; ' + f'EnvName: {self.env_name}; EnvVersion: {self.env_version}' + f')' + ) + + +@dataclass +class SbomReportStorageDetails: + path: str + folder: str + size: int + + +class SbomReportStorageDetailsSchema(Schema): + class Meta: + unknown = EXCLUDE + + path = fields.String() + folder = fields.String() + size = fields.Integer() + + @post_load + def build_dto(self, data: dict[str, Any], **_) -> SbomReportStorageDetails: + return SbomReportStorageDetails(**data) + + +@dataclass +class ReportExecution: + id: int + status: str + error_message: Optional[str] = None + status_message: Optional[str] = None + storage_details: Optional[SbomReportStorageDetails] = None + + +class ReportExecutionSchema(Schema): + class Meta: + unknown = EXCLUDE + + id = fields.Integer() + status = fields.String() + error_message = fields.String(allow_none=True) + status_message = fields.String(allow_none=True) + storage_details = fields.Nested(SbomReportStorageDetailsSchema, allow_none=True) + + @post_load + def build_dto(self, data: dict[str, Any], **_) -> ReportExecution: + return ReportExecution(**data) + + +@dataclass +class SbomReport: + report_executions: list[ReportExecution] + + +class RequestedSbomReportResultSchema(Schema): + class Meta: + unknown = EXCLUDE + + report_executions = fields.List(fields.Nested(ReportExecutionSchema)) + + @post_load + def build_dto(self, data: dict[str, Any], **_) -> SbomReport: + return SbomReport(**data) + + +@dataclass +class Member: + external_id: str + + +class MemberSchema(Schema): + class Meta: + unknown = EXCLUDE + + external_id = fields.String() + + @post_load + def build_dto(self, data: dict[str, Any], **_) -> Member: + return Member(**data) + + +@dataclass +class MemberDetails: + items: list[Member] + page_size: int + next_page_token: Optional[str] + + +class RequestedMemberDetailsResultSchema(Schema): + class Meta: + unknown = EXCLUDE + + items = fields.List(fields.Nested(MemberSchema)) + page_size = fields.Integer() + next_page_token = fields.String(allow_none=True) + + @post_load + def build_dto(self, data: dict[str, Any], **_) -> MemberDetails: + return MemberDetails(**data) + + +@dataclass +class ClassificationData: + severity: str + + +class ClassificationDataSchema(Schema): + class Meta: + unknown = EXCLUDE + + severity = fields.String() + + @post_load + def build_dto(self, data: dict[str, Any], **_) -> ClassificationData: + return ClassificationData(**data) + + +@dataclass +class DetectionRule: + classification_data: list[ClassificationData] + detection_rule_id: str + custom_remediation_guidelines: Optional[str] = None + remediation_guidelines: Optional[str] = None + description: Optional[str] = None + policy_name: Optional[str] = None + display_name: Optional[str] = None + + +class DetectionRuleSchema(Schema): + class Meta: + unknown = EXCLUDE + + classification_data = fields.Nested(ClassificationDataSchema, many=True) + detection_rule_id = fields.String() + custom_remediation_guidelines = fields.String(allow_none=True) + remediation_guidelines = fields.String(allow_none=True) + description = fields.String(allow_none=True) + policy_name = fields.String(allow_none=True) + display_name = fields.String(allow_none=True) + + @post_load + def build_dto(self, data: dict[str, Any], **_) -> DetectionRule: + return DetectionRule(**data) + + +@dataclass +class ScanResultsSyncFlow: + id: str + detection_messages: list[dict] + + +class ScanResultsSyncFlowSchema(Schema): + class Meta: + unknown = EXCLUDE + + id = fields.String() + detection_messages = fields.List(fields.Dict()) + + @post_load + def build_dto(self, data: dict[str, Any], **_) -> ScanResultsSyncFlow: + return ScanResultsSyncFlow(**data) + + +@dataclass +class SupportedModulesPreferences: + secret_scanning: bool + leak_scanning: bool + iac_scanning: bool + sca_scanning: bool + ci_cd_scanning: bool + sast_scanning: bool + container_scanning: bool + access_review: bool + asoc: bool + cimon: bool + ai_machine_learning: bool + ai_large_language_model: bool + + +class SupportedModulesPreferencesSchema(Schema): + class Meta: + unknown = EXCLUDE + + secret_scanning = fields.Boolean() + leak_scanning = fields.Boolean() + iac_scanning = fields.Boolean() + sca_scanning = fields.Boolean() + ci_cd_scanning = fields.Boolean() + sast_scanning = fields.Boolean() + container_scanning = fields.Boolean() + access_review = fields.Boolean() + asoc = fields.Boolean() + cimon = fields.Boolean() + ai_machine_learning = fields.Boolean() + ai_large_language_model = fields.Boolean() + + @post_load + def build_dto(self, data: dict[str, Any], **_) -> 'SupportedModulesPreferences': + return SupportedModulesPreferences(**data) + + +@dataclass +class ScanConfiguration: + scannable_extensions: list[str] + is_cycode_ignore_allowed: bool + + +class ScanConfigurationSchema(Schema): + class Meta: + unknown = EXCLUDE + + scannable_extensions = fields.List(fields.String(), allow_none=True) + is_cycode_ignore_allowed = fields.Boolean(load_default=True) + + @post_load + def build_dto(self, data: dict[str, Any], **_) -> 'ScanConfiguration': + return ScanConfiguration(**data) diff --git a/cycode/cyclient/report_client.py b/cycode/cyclient/report_client.py new file mode 100644 index 00000000..a55b5c40 --- /dev/null +++ b/cycode/cyclient/report_client.py @@ -0,0 +1,109 @@ +import dataclasses +import json +from typing import Optional + +from requests import Response + +from cycode.cli.exceptions.custom_exceptions import CycodeError +from cycode.cli.files_collector.models.in_memory_zip import InMemoryZip +from cycode.cli.utils.url_utils import sanitize_repository_url +from cycode.cyclient import models +from cycode.cyclient.cycode_client_base import CycodeClientBase +from cycode.logger import get_logger + +logger = get_logger('Report Client') + + +@dataclasses.dataclass +class ReportParameters: + entity_type: str + sbom_report_type: str + sbom_version: str + output_format: str + include_vulnerabilities: bool + include_dev_dependencies: bool + + def to_dict(self, *, without_entity_type: bool) -> dict: + model_dict = dataclasses.asdict(self) + if without_entity_type: + del model_dict['entity_type'] + return model_dict + + def to_json(self, *, without_entity_type: bool) -> str: + return json.dumps(self.to_dict(without_entity_type=without_entity_type)) + + +class ReportClient: + SERVICE_NAME: str = 'report' + CREATE_SBOM_REPORT_REQUEST_PATH: str = 'api/v2/report/{report_type}/sbom' + GET_EXECUTIONS_STATUS_PATH: str = 'api/v2/report/executions' + REPORT_STATUS_PATH: str = 'api/v2/report/{report_execution_id}/status' + + DOWNLOAD_REPORT_PATH: str = 'files/api/v1/file/sbom/{file_name}' # not in the report service + + def __init__(self, client: CycodeClientBase) -> None: + self.client = client + + def request_sbom_report_execution( + self, params: ReportParameters, zip_file: InMemoryZip = None, repository_url: Optional[str] = None + ) -> models.ReportExecution: + report_type = 'zipped-file' if zip_file else 'repository-url' + url_path = f'{self.SERVICE_NAME}/{self.CREATE_SBOM_REPORT_REQUEST_PATH}'.format(report_type=report_type) + + # entity type required only for zipped-file + request_data = {'report_parameters': params.to_json(without_entity_type=zip_file is None)} + if repository_url: + # Sanitize repository URL to remove any embedded credentials/tokens before sending to API + sanitized_url = sanitize_repository_url(repository_url) + if sanitized_url != repository_url: + logger.debug('Sanitized repository URL to remove credentials') + request_data['repository_url'] = sanitized_url + + request_args = { + 'url_path': url_path, + 'data': request_data, + } + + if zip_file: + request_args['files'] = {'file': ('sca_files.zip', zip_file.read())} + + response = self.client.post(**request_args) + sbom_report = self.parse_requested_sbom_report_response(response) + if not sbom_report.report_executions: + raise CycodeError('Failed to get SBOM report. No executions found.') + + return sbom_report.report_executions[0] + + def get_report_execution(self, report_execution_id: int) -> models.ReportExecutionSchema: + url_path = f'{self.SERVICE_NAME}/{self.GET_EXECUTIONS_STATUS_PATH}' + params = { + 'executions_ids': report_execution_id, + 'include_orphan_executions': True, + } + response = self.client.get(url_path=url_path, params=params) + + report_executions = self.parse_execution_status_response(response) + if not report_executions: + raise CycodeError('Failed to get report execution.') + + return report_executions[0] + + def get_file_content(self, file_name: str) -> str: + response = self.client.get( + url_path=self.DOWNLOAD_REPORT_PATH.format(file_name=file_name), + params={'include_hidden': True}, + hide_response_content_log=True, + ) + return response.text + + def report_status(self, report_execution_id: int, status: dict) -> None: + url_path = f'{self.SERVICE_NAME}/{self.REPORT_STATUS_PATH}'.format(report_execution_id=report_execution_id) + self.client.post(url_path=url_path, body=status) + + @staticmethod + def parse_requested_sbom_report_response(response: Response) -> models.SbomReport: + return models.RequestedSbomReportResultSchema().load(response.json()) + + @staticmethod + def parse_execution_status_response(response: Response) -> list[models.ReportExecutionSchema]: + return models.ReportExecutionSchema().load(response.json(), many=True) diff --git a/cycode/cyclient/scan_client.py b/cycode/cyclient/scan_client.py index 3eb29cf3..24c5ac46 100644 --- a/cycode/cyclient/scan_client.py +++ b/cycode/cyclient/scan_client.py @@ -1,122 +1,367 @@ import json -from typing import List +from copy import deepcopy +from typing import TYPE_CHECKING, Optional, Union +from uuid import UUID +import requests from requests import Response -from cycode.cli.zip_file import InMemoryZip -from . import models -from .cycode_client_base import CycodeClientBase -from .scan_config.scan_config_base import ScanConfigBase +from cycode.cli import consts +from cycode.cli.config import configuration_manager +from cycode.cli.exceptions.custom_exceptions import CycodeError, RequestHttpError +from cycode.cli.files_collector.models.in_memory_zip import InMemoryZip +from cycode.cyclient import models +from cycode.cyclient.cycode_client_base import CycodeClientBase +from cycode.cyclient.logger import logger + +if TYPE_CHECKING: + from cycode.cyclient.scan_config_base import ScanConfigBase class ScanClient: - def __init__(self, scan_cycode_client: CycodeClientBase, scan_config: ScanConfigBase): + def __init__( + self, scan_cycode_client: CycodeClientBase, scan_config: 'ScanConfigBase', hide_response_log: bool = True + ) -> None: self.scan_cycode_client = scan_cycode_client self.scan_config = scan_config - self.SCAN_CONTROLLER_PATH = 'api/v1/scan' - self.DETECTIONS_SERVICE_CONTROLLER_PATH = 'api/v1/detections' + + self._SCAN_SERVICE_CLI_CONTROLLER_PATH = 'api/v1/cli-scan' + self._SCAN_SERVICE_V4_CLI_CONTROLLER_PATH = 'api/v4/scans/cli' + self._DETECTIONS_SERVICE_CLI_CONTROLLER_PATH = 'api/v1/detections/cli' + self._POLICIES_SERVICE_CONTROLLER_PATH_V3 = 'api/v3/policies' + + self._hide_response_log = hide_response_log + + @staticmethod + def get_scan_flow_type(should_use_sync_flow: bool = False) -> str: + if should_use_sync_flow: + return '/sync' + + return '' + + def get_scan_service_url_path(self, scan_type: str, should_use_sync_flow: bool = False) -> str: + service_path = self.scan_config.get_service_name(scan_type) + flow_type = self.get_scan_flow_type(should_use_sync_flow) + return f'{service_path}/{self._SCAN_SERVICE_CLI_CONTROLLER_PATH}{flow_type}' def content_scan(self, scan_type: str, file_name: str, content: str, is_git_diff: bool = True) -> models.ScanResult: - path = f'{self.scan_config.get_service_name(scan_type)}/{self.SCAN_CONTROLLER_PATH}/content' + path = f'{self.get_scan_service_url_path(scan_type)}/content' body = {'name': file_name, 'content': content, 'is_git_diff': is_git_diff} - response = self.scan_cycode_client.post(url_path=path, body=body) + response = self.scan_cycode_client.post( + url_path=path, body=body, hide_response_content_log=self._hide_response_log + ) return self.parse_scan_response(response) - def file_scan(self, scan_type: str, path: str) -> models.ScanResult: - url_path = f'{self.scan_config.get_service_name(scan_type)}/{self.SCAN_CONTROLLER_PATH}' - files = {'file': open(path, 'rb')} - response = self.scan_cycode_client.post(url_path=url_path, files=files) - return self.parse_scan_response(response) + def get_scan_aggregation_report_url(self, aggregation_id: str, scan_type: str) -> models.ScanReportUrlResponse: + response = self.scan_cycode_client.get( + url_path=self.get_scan_aggregation_report_url_path(aggregation_id, scan_type) + ) + return models.ScanReportUrlResponseSchema().build_dto(response.json()) + + def get_scan_service_v4_url_path(self, scan_type: str) -> str: + service_path = self.scan_config.get_service_name(scan_type) + return f'{service_path}/{self._SCAN_SERVICE_V4_CLI_CONTROLLER_PATH}' + + def get_zipped_file_scan_async_url_path(self, scan_type: str, should_use_sync_flow: bool = False) -> str: + async_scan_type = self.scan_config.get_async_scan_type(scan_type) + async_entity_type = self.scan_config.get_async_entity_type(scan_type) + scan_service_url_path = self.get_scan_service_url_path(scan_type, should_use_sync_flow=should_use_sync_flow) + return f'{scan_service_url_path}/{async_scan_type}/{async_entity_type}' - def zipped_file_scan(self, scan_type: str, zip_file: InMemoryZip, scan_id: str, scan_parameters: dict, - is_git_diff: bool = False) -> models.ZippedFileScanResult: - url_path = f'{self.scan_config.get_service_name(scan_type)}/{self.SCAN_CONTROLLER_PATH}/zipped-file' + def get_zipped_file_scan_sync_url_path(self, scan_type: str) -> str: + server_scan_type = self.scan_config.get_async_scan_type(scan_type) + scan_service_url_path = self.get_scan_service_url_path(scan_type, should_use_sync_flow=True) + return f'{scan_service_url_path}/{server_scan_type}/repository' + + def zipped_file_scan_sync( + self, + zip_file: InMemoryZip, + scan_type: str, + scan_parameters: dict, + is_git_diff: bool = False, + ) -> models.ScanResultsSyncFlow: files = {'file': ('multiple_files_scan.zip', zip_file.read())} + scan_parameters = deepcopy(scan_parameters) # avoid mutating the original dict + if 'report' in scan_parameters: + del scan_parameters['report'] # BE raises validation error instead of ignoring it + response = self.scan_cycode_client.post( - url_path=url_path, - data={'scan_id': scan_id, 'is_git_diff': is_git_diff, 'scan_parameters': json.dumps(scan_parameters)}, - files=files + url_path=self.get_zipped_file_scan_sync_url_path(scan_type), + data={ + 'is_git_diff': is_git_diff, + 'scan_parameters': json.dumps(scan_parameters), + }, + files=files, + hide_response_content_log=self._hide_response_log, + timeout=configuration_manager.get_sync_scan_timeout_in_seconds(), + ) + return models.ScanResultsSyncFlowSchema().load(response.json()) + + @staticmethod + def _create_compression_manifest_string(zip_file: InMemoryZip) -> str: + return json.dumps( + { + 'file_count_by_extension': zip_file.extension_statistics, + 'file_count': zip_file.files_count, + } ) - return self.parse_zipped_file_scan_response(response) - def zipped_file_scan_async(self, zip_file: InMemoryZip, scan_type: str, scan_parameters: dict, - is_git_diff: bool = False) -> models.ScanInitializationResponse: - url_path = f'{self.scan_config.get_scans_prefix()}/{self.SCAN_CONTROLLER_PATH}/{scan_type}/repository' + def zipped_file_scan_async( + self, + zip_file: InMemoryZip, + scan_type: str, + scan_parameters: dict, + is_git_diff: bool = False, + is_commit_range: bool = False, + ) -> models.ScanInitializationResponse: files = {'file': ('multiple_files_scan.zip', zip_file.read())} + + response = self.scan_cycode_client.post( + url_path=self.get_zipped_file_scan_async_url_path(scan_type), + data={ + 'is_git_diff': is_git_diff, + 'scan_parameters': json.dumps(scan_parameters), + 'is_commit_range': is_commit_range, + 'compression_manifest': self._create_compression_manifest_string(zip_file), + }, + files=files, + ) + return models.ScanInitializationResponseSchema().load(response.json()) + + def get_upload_link(self, scan_type: str) -> models.UploadLinkResponse: + async_scan_type = self.scan_config.get_async_scan_type(scan_type) + url_path = f'{self.get_scan_service_v4_url_path(scan_type)}/{async_scan_type}/upload-link' + response = self.scan_cycode_client.get(url_path=url_path, hide_response_content_log=self._hide_response_log) + return models.UploadLinkResponseSchema().load(response.json()) + + def upload_to_presigned_post(self, url: str, fields: dict[str, str], zip_file: 'InMemoryZip') -> None: + multipart = {key: (None, value) for key, value in fields.items()} + multipart['file'] = (None, zip_file.read()) + # We are not using Cycode client, as we are calling aws S3. + response = requests.post(url, files=multipart, timeout=self.scan_cycode_client.timeout) + response.raise_for_status() + + def scan_repository_from_upload_id( + self, + scan_type: str, + upload_id: str, + scan_parameters: dict, + is_git_diff: bool = False, + is_commit_range: bool = False, + ) -> models.ScanInitializationResponse: + async_scan_type = self.scan_config.get_async_scan_type(scan_type) + url_path = f'{self.get_scan_service_v4_url_path(scan_type)}/{async_scan_type}/repository' response = self.scan_cycode_client.post( url_path=url_path, - data={'is_git_diff': is_git_diff, 'scan_parameters': json.dumps(scan_parameters)}, - files=files + body={ + 'upload_id': upload_id, + 'is_git_diff': is_git_diff, + 'is_commit_range': is_commit_range, + 'scan_parameters': json.dumps(scan_parameters), + }, ) return models.ScanInitializationResponseSchema().load(response.json()) - def multiple_zipped_file_scan_async(self, from_commit_zip_file: InMemoryZip, to_commit_zip_file: InMemoryZip, - scan_type: str, scan_parameters: dict, - is_git_diff: bool = False) -> models.ScanInitializationResponse: - url_path = f'{self.scan_config.get_scans_prefix()}/{self.SCAN_CONTROLLER_PATH}/{scan_type}/repository/commit-range' + def commit_range_scan_async( + self, + from_commit_zip_file: InMemoryZip, + to_commit_zip_file: InMemoryZip, + scan_type: str, + scan_parameters: dict, + is_git_diff: bool = False, + ) -> models.ScanInitializationResponse: + """Commit range scan. + Used by SCA and SAST scans. + + For SCA: + - from_commit_zip_file is file content + - to_commit_zip_file is file content + + For SAST: + - from_commit_zip_file is file content + - to_commit_zip_file is diff content + + Note: + Compression manifest is supported only for SAST scans. + """ + url_path = f'{self.get_scan_service_url_path(scan_type)}/{scan_type}/repository/commit-range' files = { 'file_from_commit': ('multiple_files_scan.zip', from_commit_zip_file.read()), - 'file_to_commit': ('multiple_files_scan.zip', to_commit_zip_file.read()) + 'file_to_commit': ('multiple_files_scan.zip', to_commit_zip_file.read()), } response = self.scan_cycode_client.post( url_path=url_path, - data={'is_git_diff': is_git_diff, 'scan_parameters': json.dumps(scan_parameters)}, - files=files + data={ + 'is_git_diff': is_git_diff, + 'scan_parameters': json.dumps(scan_parameters), + 'compression_manifest': self._create_compression_manifest_string(from_commit_zip_file), + }, + files=files, ) return models.ScanInitializationResponseSchema().load(response.json()) - def get_scan_details(self, scan_id: str) -> models.ScanDetailsResponse: - url_path = f'{self.scan_config.get_scans_prefix()}/{self.SCAN_CONTROLLER_PATH}/{scan_id}' - response = self.scan_cycode_client.get(url_path=url_path) + def commit_range_scan_from_upload_ids( + self, + scan_type: str, + from_commit_upload_id: str, + to_commit_upload_id: str, + scan_parameters: dict, + is_git_diff: bool = False, + ) -> models.ScanInitializationResponse: + async_scan_type = self.scan_config.get_async_scan_type(scan_type) + url_path = f'{self.get_scan_service_v4_url_path(scan_type)}/{async_scan_type}/commit-range' + response = self.scan_cycode_client.post( + url_path=url_path, + body={ + 'from_commit_upload_id': from_commit_upload_id, + 'to_commit_upload_id': to_commit_upload_id, + 'is_git_diff': is_git_diff, + 'scan_parameters': json.dumps(scan_parameters), + }, + ) + return models.ScanInitializationResponseSchema().load(response.json()) + + def get_scan_details_path(self, scan_type: str, scan_id: str) -> str: + return f'{self.get_scan_service_url_path(scan_type)}/{scan_id}' + + def get_scan_aggregation_report_url_path(self, aggregation_id: str, scan_type: str) -> str: + return f'{self.get_scan_service_url_path(scan_type)}/reportUrlByAggregationId/{aggregation_id}' + + def get_scan_details(self, scan_type: str, scan_id: str) -> models.ScanDetailsResponse: + path = self.get_scan_details_path(scan_type, scan_id) + response = self.scan_cycode_client.get(url_path=path) return models.ScanDetailsResponseSchema().load(response.json()) - def get_scan_detections(self, scan_id: str) -> List[dict]: - detections = [] - page_number = 0 + def get_detection_rules_path(self) -> str: + return ( + f'{self.scan_config.get_detections_prefix()}/' + f'{self._POLICIES_SERVICE_CONTROLLER_PATH_V3}/' + f'detection_rules/byIds' + ) + + def get_supported_modules_preferences(self) -> models.SupportedModulesPreferences: + response = self.scan_cycode_client.get(url_path='preferences/api/v1/supportedmodules') + return models.SupportedModulesPreferencesSchema().load(response.json()) + + @staticmethod + def get_ai_remediation_path(detection_id: str) -> str: + return f'scm-remediator/api/v1/ContentRemediation/preview/{detection_id}' + + def get_ai_remediation(self, detection_id: UUID, *, fix: bool = False) -> str: + path = self.get_ai_remediation_path(detection_id.hex) + + data = { + 'resolving_parameters': { + 'get_diff': True, + 'use_code_snippet': True, + 'add_diff_header': True, + } + } + if not fix: + data['resolving_parameters']['remediation_action'] = 'ReplyWithRemediationDetails' + + response = self.scan_cycode_client.get( + url_path=path, json=data, timeout=configuration_manager.get_ai_remediation_timeout_in_seconds() + ) + return response.text.strip() + + @staticmethod + def _get_policy_type_by_scan_type(scan_type: str) -> str: + scan_type_to_policy_type = { + consts.IAC_SCAN_TYPE: 'IaC', + consts.SCA_SCAN_TYPE: 'SCA', + consts.SECRET_SCAN_TYPE: 'SecretDetection', + consts.SAST_SCAN_TYPE: 'SAST', + } + + if scan_type not in scan_type_to_policy_type: + raise CycodeError('Invalid scan type') + + return scan_type_to_policy_type[scan_type] + + @staticmethod + def parse_detection_rules_response(response: Response) -> list[models.DetectionRule]: + return models.DetectionRuleSchema().load(response.json(), many=True) + + def get_detection_rules(self, detection_rules_ids: Union[set[str], list[str]]) -> list[models.DetectionRule]: + response = self.scan_cycode_client.get( + url_path=self.get_detection_rules_path(), + params={'ids': detection_rules_ids}, + hide_response_content_log=self._hide_response_log, + ) + + return self.parse_detection_rules_response(response) + + def get_scan_detections_path(self) -> str: + return f'{self.scan_config.get_detections_prefix()}/{self._DETECTIONS_SERVICE_CLI_CONTROLLER_PATH}' + + def get_scan_detections_list_path(self) -> str: + return f'{self.get_scan_detections_path()}/detections' + + def get_scan_raw_detections(self, scan_id: str) -> list[dict]: + params = {'scan_id': scan_id} + page_size = 200 - last_response_size = 0 + raw_detections = [] + + page_number = 0 + last_response_size = 0 while page_number == 0 or last_response_size == page_size: - url_path = f'{self.scan_config.get_detections_prefix()}/{self.DETECTIONS_SERVICE_CONTROLLER_PATH}?scan_id={scan_id}&page_size={page_size}&page_number={page_number}' - response = self.scan_cycode_client.get(url_path=url_path).json() - detections.extend(response) + params['page_size'] = page_size + params['page_number'] = page_number + + response = self.scan_cycode_client.get( + url_path=self.get_scan_detections_list_path(), + params=params, + hide_response_content_log=self._hide_response_log, + ).json() + raw_detections.extend(response) page_number += 1 last_response_size = len(response) - return detections - - def get_scan_detections_count(self, scan_id: str) -> int: - url_path = f'{self.scan_config.get_detections_prefix()}/{self.DETECTIONS_SERVICE_CONTROLLER_PATH}/count?scan_id={scan_id}' - response = self.scan_cycode_client.get(url_path=url_path) - return response.json().get('count', 0) + return raw_detections - def commit_range_zipped_file_scan( - self, scan_type: str, zip_file: InMemoryZip, scan_id: str - ) -> models.ZippedFileScanResult: - url_path = f'{self.scan_config.get_service_name(scan_type)}/{self.SCAN_CONTROLLER_PATH}/commit-range-zipped-file' - files = {'file': ('multiple_files_scan.zip', zip_file.read())} - response = self.scan_cycode_client.post(url_path=url_path, data={'scan_id': scan_id}, files=files) - return self.parse_zipped_file_scan_response(response) + def get_report_scan_status_path(self, scan_type: str, scan_id: str) -> str: + return f'{self.get_scan_service_url_path(scan_type)}/{scan_id}/status' - def report_scan_status(self, scan_type: str, scan_id: str, scan_status: dict): - url_path = f'{self.scan_config.get_service_name(scan_type)}/{self.SCAN_CONTROLLER_PATH}/{scan_id}/status' - self.scan_cycode_client.post(url_path=url_path, body=scan_status) + def report_scan_status(self, scan_type: str, scan_id: str, scan_status: dict) -> None: + self.scan_cycode_client.post( + url_path=self.get_report_scan_status_path(scan_type, scan_id), + body=scan_status, + ) @staticmethod def parse_scan_response(response: Response) -> models.ScanResult: return models.ScanResultSchema().load(response.json()) - @staticmethod - def parse_zipped_file_scan_response(response: Response) -> models.ZippedFileScanResult: - return models.ZippedFileScanResultSchema().load(response.json()) + def get_scan_configuration_path(self, scan_type: str) -> str: + correct_scan_type = self.scan_config.get_async_scan_type(scan_type) + return f'{self.get_scan_service_url_path(scan_type)}/{correct_scan_type}/configuration' - @staticmethod - def get_service_name(scan_type: str) -> str: - if scan_type == 'secret': - return 'secret' - elif scan_type == 'iac': - return 'iac' - elif scan_type == 'sca' or scan_type == 'sast': - return 'scans' + def get_scan_configuration(self, scan_type: str, remote_url: Optional[str] = None) -> models.ScanConfiguration: + params = {} + if remote_url: + params['remote_url'] = remote_url + + response = self.scan_cycode_client.get( + url_path=self.get_scan_configuration_path(scan_type), + params=params, + hide_response_content_log=self._hide_response_log, + ) + return models.ScanConfigurationSchema().load(response.json()) + + def get_scan_configuration_safe( + self, scan_type: str, remote_url: Optional[str] = None + ) -> Optional['models.ScanConfiguration']: + try: + return self.get_scan_configuration(scan_type, remote_url) + except RequestHttpError as e: + if e.status_code == 404: + logger.debug( + 'Remote scan configuration is not supported for this scan type: %s', {'scan_type': scan_type} + ) + else: + logger.debug('Failed to get remote scan configuration: %s', {'scan_type': scan_type}, exc_info=e) diff --git a/cycode/cyclient/scan_config/scan_config_base.py b/cycode/cyclient/scan_config/scan_config_base.py deleted file mode 100644 index 957923d4..00000000 --- a/cycode/cyclient/scan_config/scan_config_base.py +++ /dev/null @@ -1,50 +0,0 @@ -from abc import ABC, abstractmethod - - -class ScanConfigBase(ABC): - - @abstractmethod - def get_service_name(self, scan_type): - pass - - @abstractmethod - def get_scans_prefix(self): - pass - - @abstractmethod - def get_detections_prefix(self): - pass - - -class DevScanConfig(ScanConfigBase): - - def get_service_name(self, scan_type): - if scan_type == 'secret': - return '5025' - elif scan_type == 'iac': - return '5026' - elif scan_type == 'sca' or scan_type == 'sast': - return '5004' - - def get_scans_prefix(self): - return '5004' - - def get_detections_prefix(self): - return '5016' - - -class DefaultScanConfig(ScanConfigBase): - - def get_service_name(self, scan_type): - if scan_type == 'secret': - return 'secret' - elif scan_type == 'iac': - return 'iac' - elif scan_type == 'sca' or scan_type == 'sast': - return 'scans' - - def get_scans_prefix(self): - return 'scans' - - def get_detections_prefix(self): - return 'detections' diff --git a/cycode/cyclient/scan_config/scan_config_creator.py b/cycode/cyclient/scan_config/scan_config_creator.py deleted file mode 100644 index 5f0b8a7c..00000000 --- a/cycode/cyclient/scan_config/scan_config_creator.py +++ /dev/null @@ -1,28 +0,0 @@ -from ..config import dev_mode -from ..config_dev import DEV_CYCODE_API_URL -from ..cycode_dev_based_client import CycodeDevBasedClient -from ..cycode_token_based_client import CycodeTokenBasedClient -from ..scan_client import ScanClient -from ..scan_config.scan_config_base import DefaultScanConfig, DevScanConfig - - -def create_scan_client(client_id, client_secret): - if dev_mode: - scan_cycode_client, scan_config = create_scan_for_dev_env() - else: - scan_cycode_client, scan_config = create_scan(client_id, client_secret) - - return ScanClient(scan_cycode_client=scan_cycode_client, - scan_config=scan_config) - - -def create_scan(client_id, client_secret): - scan_cycode_client = CycodeTokenBasedClient(client_id, client_secret) - scan_config = DefaultScanConfig() - return scan_cycode_client, scan_config - - -def create_scan_for_dev_env(): - scan_cycode_client = CycodeDevBasedClient(DEV_CYCODE_API_URL) - scan_config = DevScanConfig() - return scan_cycode_client, scan_config diff --git a/cycode/cyclient/scan_config_base.py b/cycode/cyclient/scan_config_base.py new file mode 100644 index 00000000..d60068ce --- /dev/null +++ b/cycode/cyclient/scan_config_base.py @@ -0,0 +1,43 @@ +from abc import ABC, abstractmethod + +from cycode.cli import consts + + +class ScanConfigBase(ABC): + @abstractmethod + def get_service_name(self, scan_type: str) -> str: ... + + @staticmethod + def get_async_scan_type(scan_type: str) -> str: + if scan_type == consts.SECRET_SCAN_TYPE: + return 'Secrets' + if scan_type == consts.IAC_SCAN_TYPE: + return 'InfraConfiguration' + + return scan_type.upper() + + @staticmethod + def get_async_entity_type(scan_type: str) -> str: + if scan_type == consts.SECRET_SCAN_TYPE: + return 'ZippedFile' + # we are migrating to "zippedfile" entity type. will be used later + return 'repository' + + @abstractmethod + def get_detections_prefix(self) -> str: ... + + +class DevScanConfig(ScanConfigBase): + def get_service_name(self, scan_type: str) -> str: + return '5004' # scan service + + def get_detections_prefix(self) -> str: + return '5016' # detections service + + +class DefaultScanConfig(ScanConfigBase): + def get_service_name(self, scan_type: str) -> str: + return 'scans' # scan service + + def get_detections_prefix(self) -> str: + return 'detections' diff --git a/cycode/cyclient/utils.py b/cycode/cyclient/utils.py deleted file mode 100644 index b8be58dc..00000000 --- a/cycode/cyclient/utils.py +++ /dev/null @@ -1,10 +0,0 @@ -import os - - -def split_list(input_list, batch_size): - for i in range(0, len(input_list), batch_size): - yield input_list[i:i + batch_size] - - -def cpu_count(): - return os.cpu_count() or 1 diff --git a/cycode/logger.py b/cycode/logger.py new file mode 100644 index 00000000..c5cdebcf --- /dev/null +++ b/cycode/logger.py @@ -0,0 +1,70 @@ +import logging +import sys +from typing import ClassVar, NamedTuple, Optional, Union + +import click +import typer +from rich.logging import RichHandler + +from cycode.cli import consts +from cycode.cli.console import console_err +from cycode.config import get_val_as_string + + +def _set_io_encodings() -> None: + # set io encoding (for Windows) + sys.stdout.reconfigure(encoding='UTF-8') + sys.stderr.reconfigure(encoding='UTF-8') + + +_set_io_encodings() + +_RICH_LOGGING_HANDLER = RichHandler(console=console_err, rich_tracebacks=True, tracebacks_suppress=[click, typer]) + +logging.basicConfig( + level=logging.INFO, + format='[%(name)s] %(message)s', + handlers=[_RICH_LOGGING_HANDLER], +) + +logging.getLogger('urllib3').setLevel(logging.WARNING) +logging.getLogger('werkzeug').setLevel(logging.WARNING) +logging.getLogger('schedule').setLevel(logging.WARNING) +logging.getLogger('kubernetes').setLevel(logging.WARNING) +logging.getLogger('git.cmd').setLevel(logging.WARNING) +logging.getLogger('git.util').setLevel(logging.WARNING) + + +class CreatedLogger(NamedTuple): + logger: logging.Logger + control_level_in_runtime: bool + + +class LoggersManager: + loggers: ClassVar[set[CreatedLogger]] = set() + global_logging_level: Optional[int] = None + + +def get_logger_level() -> Optional[Union[int, str]]: + if LoggersManager.global_logging_level is not None: + return LoggersManager.global_logging_level + + config_level = get_val_as_string(consts.LOGGING_LEVEL_ENV_VAR_NAME) + return logging.getLevelName(config_level) + + +def get_logger(logger_name: Optional[str] = None, control_level_in_runtime: bool = True) -> logging.Logger: + new_logger = logging.getLogger(logger_name) + new_logger.setLevel(get_logger_level()) + + LoggersManager.loggers.add(CreatedLogger(logger=new_logger, control_level_in_runtime=control_level_in_runtime)) + + return new_logger + + +def set_logging_level(level: int) -> None: + LoggersManager.global_logging_level = level + + for created_logger in LoggersManager.loggers: + if created_logger.control_level_in_runtime: + created_logger.logger.setLevel(level) diff --git a/images/sca_report_url.png b/images/sca_report_url.png new file mode 100644 index 00000000..6513966e Binary files /dev/null and b/images/sca_report_url.png differ diff --git a/poetry.lock b/poetry.lock index 51b3c7d7..0bd73e47 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,179 +1,340 @@ -# This file is automatically @generated by Poetry and should not be changed by hand. +# This file is automatically @generated by Poetry 2.2.1 and should not be changed by hand. [[package]] name = "altgraph" -version = "0.17.3" +version = "0.17.4" description = "Python graph (network) package" -category = "dev" optional = false python-versions = "*" +groups = ["executable"] +markers = "python_version < \"3.15\"" files = [ - {file = "altgraph-0.17.3-py2.py3-none-any.whl", hash = "sha256:c8ac1ca6772207179ed8003ce7687757c04b0b71536f81e2ac5755c6226458fe"}, - {file = "altgraph-0.17.3.tar.gz", hash = "sha256:ad33358114df7c9416cdb8fa1eaa5852166c505118717021c6a8c7c7abbd03dd"}, + {file = "altgraph-0.17.4-py2.py3-none-any.whl", hash = "sha256:642743b4750de17e655e6711601b077bc6598dbfa3ba5fa2b2a35ce12b508dff"}, + {file = "altgraph-0.17.4.tar.gz", hash = "sha256:1b5afbb98f6c4dcadb2e2ae6ab9fa994bbb8c1d75f4fa96d340f9437ae454406"}, ] +[[package]] +name = "annotated-types" +version = "0.7.0" +description = "Reusable constraint types to use with typing.Annotated" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53"}, + {file = "annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89"}, +] + +[[package]] +name = "anyio" +version = "4.11.0" +description = "High-level concurrency and networking framework on top of asyncio or Trio" +optional = false +python-versions = ">=3.9" +groups = ["main"] +markers = "python_version >= \"3.10\"" +files = [ + {file = "anyio-4.11.0-py3-none-any.whl", hash = "sha256:0287e96f4d26d4149305414d4e3bc32f0dcd0862365a4bddea19d7a1ec38c4fc"}, + {file = "anyio-4.11.0.tar.gz", hash = "sha256:82a8d0b81e318cc5ce71a5f1f8b5c4e63619620b63141ef8c995fa0db95a57c4"}, +] + +[package.dependencies] +exceptiongroup = {version = ">=1.0.2", markers = "python_version < \"3.11\""} +idna = ">=2.8" +sniffio = ">=1.1" +typing_extensions = {version = ">=4.5", markers = "python_version < \"3.13\""} + +[package.extras] +trio = ["trio (>=0.31.0)"] + [[package]] name = "arrow" -version = "0.17.0" +version = "1.4.0" description = "Better dates & times for Python" -category = "main" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +python-versions = ">=3.8" +groups = ["main"] files = [ - {file = "arrow-0.17.0-py2.py3-none-any.whl", hash = "sha256:e098abbd9af3665aea81bdd6c869e93af4feb078e98468dd351c383af187aac5"}, - {file = "arrow-0.17.0.tar.gz", hash = "sha256:ff08d10cda1d36c68657d6ad20d74fbea493d980f8b2d45344e00d6ed2bf6ed4"}, + {file = "arrow-1.4.0-py3-none-any.whl", hash = "sha256:749f0769958ebdc79c173ff0b0670d59051a535fa26e8eba02953dc19eb43205"}, + {file = "arrow-1.4.0.tar.gz", hash = "sha256:ed0cc050e98001b8779e84d461b0098c4ac597e88704a655582b21d116e526d7"}, ] [package.dependencies] python-dateutil = ">=2.7.0" +tzdata = {version = "*", markers = "python_version >= \"3.9\""} + +[package.extras] +doc = ["doc8", "sphinx (>=7.0.0)", "sphinx-autobuild", "sphinx-autodoc-typehints", "sphinx_rtd_theme (>=1.3.0)"] +test = ["dateparser (==1.*)", "pre-commit", "pytest", "pytest-cov", "pytest-mock", "pytz (==2025.2)", "simplejson (==3.*)"] [[package]] -name = "binaryornot" -version = "0.4.4" -description = "Ultra-lightweight pure Python package to check if a file is binary or text." -category = "main" +name = "attrs" +version = "25.4.0" +description = "Classes Without Boilerplate" optional = false -python-versions = "*" +python-versions = ">=3.9" +groups = ["main"] +markers = "python_version >= \"3.10\"" files = [ - {file = "binaryornot-0.4.4-py2.py3-none-any.whl", hash = "sha256:b8b71173c917bddcd2c16070412e369c3ed7f0528926f70cac18a6c97fd563e4"}, - {file = "binaryornot-0.4.4.tar.gz", hash = "sha256:359501dfc9d40632edc9fac890e19542db1a287bbcfa58175b66658392018061"}, + {file = "attrs-25.4.0-py3-none-any.whl", hash = "sha256:adcf7e2a1fb3b36ac48d97835bb6d8ade15b8dcce26aba8bf1d14847b57a3373"}, + {file = "attrs-25.4.0.tar.gz", hash = "sha256:16d5969b87f0859ef33a48b35d55ac1be6e42ae49d5e853b597db70c35c57e11"}, ] -[package.dependencies] -chardet = ">=3.0.2" - [[package]] name = "certifi" -version = "2023.5.7" +version = "2025.10.5" description = "Python package for providing Mozilla's CA Bundle." -category = "main" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" +groups = ["main", "test"] files = [ - {file = "certifi-2023.5.7-py3-none-any.whl", hash = "sha256:c6c2e98f5c7869efca1f8916fed228dd91539f9f1b444c314c06eef02980c716"}, - {file = "certifi-2023.5.7.tar.gz", hash = "sha256:0f0d56dc5a6ad56fd4ba36484d6cc34451e1c6548c61daad8c320169f91eddc7"}, + {file = "certifi-2025.10.5-py3-none-any.whl", hash = "sha256:0f212c2744a9bb6de0c56639a6f68afe01ecd92d91f14ae897c4fe7bbeeef0de"}, + {file = "certifi-2025.10.5.tar.gz", hash = "sha256:47c09d31ccf2acf0be3f701ea53595ee7e0b8fa08801c6624be771df09ae7b43"}, ] [[package]] -name = "chardet" -version = "5.1.0" -description = "Universal encoding detector for Python 3" -category = "main" +name = "cffi" +version = "2.0.0" +description = "Foreign Function Interface for Python calling C code." optional = false -python-versions = ">=3.7" +python-versions = ">=3.9" +groups = ["main"] +markers = "python_version >= \"3.10\" and platform_python_implementation != \"PyPy\"" files = [ - {file = "chardet-5.1.0-py3-none-any.whl", hash = "sha256:362777fb014af596ad31334fde1e8c327dfdb076e1960d1694662d46a6917ab9"}, - {file = "chardet-5.1.0.tar.gz", hash = "sha256:0d62712b956bc154f85fb0a266e2a3c5913c2967e00348701b32411d6def31e5"}, + {file = "cffi-2.0.0-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:0cf2d91ecc3fcc0625c2c530fe004f82c110405f101548512cce44322fa8ac44"}, + {file = "cffi-2.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f73b96c41e3b2adedc34a7356e64c8eb96e03a3782b535e043a986276ce12a49"}, + {file = "cffi-2.0.0-cp310-cp310-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:53f77cbe57044e88bbd5ed26ac1d0514d2acf0591dd6bb02a3ae37f76811b80c"}, + {file = "cffi-2.0.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:3e837e369566884707ddaf85fc1744b47575005c0a229de3327f8f9a20f4efeb"}, + {file = "cffi-2.0.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:5eda85d6d1879e692d546a078b44251cdd08dd1cfb98dfb77b670c97cee49ea0"}, + {file = "cffi-2.0.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:9332088d75dc3241c702d852d4671613136d90fa6881da7d770a483fd05248b4"}, + {file = "cffi-2.0.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fc7de24befaeae77ba923797c7c87834c73648a05a4bde34b3b7e5588973a453"}, + {file = "cffi-2.0.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:cf364028c016c03078a23b503f02058f1814320a56ad535686f90565636a9495"}, + {file = "cffi-2.0.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e11e82b744887154b182fd3e7e8512418446501191994dbf9c9fc1f32cc8efd5"}, + {file = "cffi-2.0.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:8ea985900c5c95ce9db1745f7933eeef5d314f0565b27625d9a10ec9881e1bfb"}, + {file = "cffi-2.0.0-cp310-cp310-win32.whl", hash = "sha256:1f72fb8906754ac8a2cc3f9f5aaa298070652a0ffae577e0ea9bd480dc3c931a"}, + {file = "cffi-2.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:b18a3ed7d5b3bd8d9ef7a8cb226502c6bf8308df1525e1cc676c3680e7176739"}, + {file = "cffi-2.0.0-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:b4c854ef3adc177950a8dfc81a86f5115d2abd545751a304c5bcf2c2c7283cfe"}, + {file = "cffi-2.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2de9a304e27f7596cd03d16f1b7c72219bd944e99cc52b84d0145aefb07cbd3c"}, + {file = "cffi-2.0.0-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:baf5215e0ab74c16e2dd324e8ec067ef59e41125d3eade2b863d294fd5035c92"}, + {file = "cffi-2.0.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:730cacb21e1bdff3ce90babf007d0a0917cc3e6492f336c2f0134101e0944f93"}, + {file = "cffi-2.0.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:6824f87845e3396029f3820c206e459ccc91760e8fa24422f8b0c3d1731cbec5"}, + {file = "cffi-2.0.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:9de40a7b0323d889cf8d23d1ef214f565ab154443c42737dfe52ff82cf857664"}, + {file = "cffi-2.0.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8941aaadaf67246224cee8c3803777eed332a19d909b47e29c9842ef1e79ac26"}, + {file = "cffi-2.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:a05d0c237b3349096d3981b727493e22147f934b20f6f125a3eba8f994bec4a9"}, + {file = "cffi-2.0.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:94698a9c5f91f9d138526b48fe26a199609544591f859c870d477351dc7b2414"}, + {file = "cffi-2.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:5fed36fccc0612a53f1d4d9a816b50a36702c28a2aa880cb8a122b3466638743"}, + {file = "cffi-2.0.0-cp311-cp311-win32.whl", hash = "sha256:c649e3a33450ec82378822b3dad03cc228b8f5963c0c12fc3b1e0ab940f768a5"}, + {file = "cffi-2.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:66f011380d0e49ed280c789fbd08ff0d40968ee7b665575489afa95c98196ab5"}, + {file = "cffi-2.0.0-cp311-cp311-win_arm64.whl", hash = "sha256:c6638687455baf640e37344fe26d37c404db8b80d037c3d29f58fe8d1c3b194d"}, + {file = "cffi-2.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d02d6655b0e54f54c4ef0b94eb6be0607b70853c45ce98bd278dc7de718be5d"}, + {file = "cffi-2.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8eca2a813c1cb7ad4fb74d368c2ffbbb4789d377ee5bb8df98373c2cc0dee76c"}, + {file = "cffi-2.0.0-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:21d1152871b019407d8ac3985f6775c079416c282e431a4da6afe7aefd2bccbe"}, + {file = "cffi-2.0.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b21e08af67b8a103c71a250401c78d5e0893beff75e28c53c98f4de42f774062"}, + {file = "cffi-2.0.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:1e3a615586f05fc4065a8b22b8152f0c1b00cdbc60596d187c2a74f9e3036e4e"}, + {file = "cffi-2.0.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:81afed14892743bbe14dacb9e36d9e0e504cd204e0b165062c488942b9718037"}, + {file = "cffi-2.0.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3e17ed538242334bf70832644a32a7aae3d83b57567f9fd60a26257e992b79ba"}, + {file = "cffi-2.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3925dd22fa2b7699ed2617149842d2e6adde22b262fcbfada50e3d195e4b3a94"}, + {file = "cffi-2.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2c8f814d84194c9ea681642fd164267891702542f028a15fc97d4674b6206187"}, + {file = "cffi-2.0.0-cp312-cp312-win32.whl", hash = "sha256:da902562c3e9c550df360bfa53c035b2f241fed6d9aef119048073680ace4a18"}, + {file = "cffi-2.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:da68248800ad6320861f129cd9c1bf96ca849a2771a59e0344e88681905916f5"}, + {file = "cffi-2.0.0-cp312-cp312-win_arm64.whl", hash = "sha256:4671d9dd5ec934cb9a73e7ee9676f9362aba54f7f34910956b84d727b0d73fb6"}, + {file = "cffi-2.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:00bdf7acc5f795150faa6957054fbbca2439db2f775ce831222b66f192f03beb"}, + {file = "cffi-2.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:45d5e886156860dc35862657e1494b9bae8dfa63bf56796f2fb56e1679fc0bca"}, + {file = "cffi-2.0.0-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:07b271772c100085dd28b74fa0cd81c8fb1a3ba18b21e03d7c27f3436a10606b"}, + {file = "cffi-2.0.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d48a880098c96020b02d5a1f7d9251308510ce8858940e6fa99ece33f610838b"}, + {file = "cffi-2.0.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:f93fd8e5c8c0a4aa1f424d6173f14a892044054871c771f8566e4008eaa359d2"}, + {file = "cffi-2.0.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:dd4f05f54a52fb558f1ba9f528228066954fee3ebe629fc1660d874d040ae5a3"}, + {file = "cffi-2.0.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c8d3b5532fc71b7a77c09192b4a5a200ea992702734a2e9279a37f2478236f26"}, + {file = "cffi-2.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d9b29c1f0ae438d5ee9acb31cadee00a58c46cc9c0b2f9038c6b0b3470877a8c"}, + {file = "cffi-2.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6d50360be4546678fc1b79ffe7a66265e28667840010348dd69a314145807a1b"}, + {file = "cffi-2.0.0-cp313-cp313-win32.whl", hash = "sha256:74a03b9698e198d47562765773b4a8309919089150a0bb17d829ad7b44b60d27"}, + {file = "cffi-2.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:19f705ada2530c1167abacb171925dd886168931e0a7b78f5bffcae5c6b5be75"}, + {file = "cffi-2.0.0-cp313-cp313-win_arm64.whl", hash = "sha256:256f80b80ca3853f90c21b23ee78cd008713787b1b1e93eae9f3d6a7134abd91"}, + {file = "cffi-2.0.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:fc33c5141b55ed366cfaad382df24fe7dcbc686de5be719b207bb248e3053dc5"}, + {file = "cffi-2.0.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c654de545946e0db659b3400168c9ad31b5d29593291482c43e3564effbcee13"}, + {file = "cffi-2.0.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:24b6f81f1983e6df8db3adc38562c83f7d4a0c36162885ec7f7b77c7dcbec97b"}, + {file = "cffi-2.0.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:12873ca6cb9b0f0d3a0da705d6086fe911591737a59f28b7936bdfed27c0d47c"}, + {file = "cffi-2.0.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:d9b97165e8aed9272a6bb17c01e3cc5871a594a446ebedc996e2397a1c1ea8ef"}, + {file = "cffi-2.0.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:afb8db5439b81cf9c9d0c80404b60c3cc9c3add93e114dcae767f1477cb53775"}, + {file = "cffi-2.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:737fe7d37e1a1bffe70bd5754ea763a62a066dc5913ca57e957824b72a85e205"}, + {file = "cffi-2.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:38100abb9d1b1435bc4cc340bb4489635dc2f0da7456590877030c9b3d40b0c1"}, + {file = "cffi-2.0.0-cp314-cp314-win32.whl", hash = "sha256:087067fa8953339c723661eda6b54bc98c5625757ea62e95eb4898ad5e776e9f"}, + {file = "cffi-2.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:203a48d1fb583fc7d78a4c6655692963b860a417c0528492a6bc21f1aaefab25"}, + {file = "cffi-2.0.0-cp314-cp314-win_arm64.whl", hash = "sha256:dbd5c7a25a7cb98f5ca55d258b103a2054f859a46ae11aaf23134f9cc0d356ad"}, + {file = "cffi-2.0.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:9a67fc9e8eb39039280526379fb3a70023d77caec1852002b4da7e8b270c4dd9"}, + {file = "cffi-2.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7a66c7204d8869299919db4d5069a82f1561581af12b11b3c9f48c584eb8743d"}, + {file = "cffi-2.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7cc09976e8b56f8cebd752f7113ad07752461f48a58cbba644139015ac24954c"}, + {file = "cffi-2.0.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:92b68146a71df78564e4ef48af17551a5ddd142e5190cdf2c5624d0c3ff5b2e8"}, + {file = "cffi-2.0.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:b1e74d11748e7e98e2f426ab176d4ed720a64412b6a15054378afdb71e0f37dc"}, + {file = "cffi-2.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:28a3a209b96630bca57cce802da70c266eb08c6e97e5afd61a75611ee6c64592"}, + {file = "cffi-2.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7553fb2090d71822f02c629afe6042c299edf91ba1bf94951165613553984512"}, + {file = "cffi-2.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6c6c373cfc5c83a975506110d17457138c8c63016b563cc9ed6e056a82f13ce4"}, + {file = "cffi-2.0.0-cp314-cp314t-win32.whl", hash = "sha256:1fc9ea04857caf665289b7a75923f2c6ed559b8298a1b8c49e59f7dd95c8481e"}, + {file = "cffi-2.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:d68b6cef7827e8641e8ef16f4494edda8b36104d79773a334beaa1e3521430f6"}, + {file = "cffi-2.0.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0a1527a803f0a659de1af2e1fd700213caba79377e27e4693648c2923da066f9"}, + {file = "cffi-2.0.0-cp39-cp39-macosx_10_13_x86_64.whl", hash = "sha256:fe562eb1a64e67dd297ccc4f5addea2501664954f2692b69a76449ec7913ecbf"}, + {file = "cffi-2.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:de8dad4425a6ca6e4e5e297b27b5c824ecc7581910bf9aee86cb6835e6812aa7"}, + {file = "cffi-2.0.0-cp39-cp39-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:4647afc2f90d1ddd33441e5b0e85b16b12ddec4fca55f0d9671fef036ecca27c"}, + {file = "cffi-2.0.0-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:3f4d46d8b35698056ec29bca21546e1551a205058ae1a181d871e278b0b28165"}, + {file = "cffi-2.0.0-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:e6e73b9e02893c764e7e8d5bb5ce277f1a009cd5243f8228f75f842bf937c534"}, + {file = "cffi-2.0.0-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:cb527a79772e5ef98fb1d700678fe031e353e765d1ca2d409c92263c6d43e09f"}, + {file = "cffi-2.0.0-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:61d028e90346df14fedc3d1e5441df818d095f3b87d286825dfcbd6459b7ef63"}, + {file = "cffi-2.0.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:0f6084a0ea23d05d20c3edcda20c3d006f9b6f3fefeac38f59262e10cef47ee2"}, + {file = "cffi-2.0.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:1cd13c99ce269b3ed80b417dcd591415d3372bcac067009b6e0f59c7d4015e65"}, + {file = "cffi-2.0.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:89472c9762729b5ae1ad974b777416bfda4ac5642423fa93bd57a09204712322"}, + {file = "cffi-2.0.0-cp39-cp39-win32.whl", hash = "sha256:2081580ebb843f759b9f617314a24ed5738c51d2aee65d31e02f6f7a2b97707a"}, + {file = "cffi-2.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:b882b3df248017dba09d6b16defe9b5c407fe32fc7c65a9c69798e6175601be9"}, + {file = "cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529"}, ] +[package.dependencies] +pycparser = {version = "*", markers = "implementation_name != \"PyPy\""} + [[package]] name = "charset-normalizer" -version = "3.1.0" +version = "3.4.4" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." -category = "main" -optional = false -python-versions = ">=3.7.0" -files = [ - {file = "charset-normalizer-3.1.0.tar.gz", hash = "sha256:34e0a2f9c370eb95597aae63bf85eb5e96826d81e3dcf88b8886012906f509b5"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e0ac8959c929593fee38da1c2b64ee9778733cdf03c482c9ff1d508b6b593b2b"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d7fc3fca01da18fbabe4625d64bb612b533533ed10045a2ac3dd194bfa656b60"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:04eefcee095f58eaabe6dc3cc2262f3bcd776d2c67005880894f447b3f2cb9c1"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:20064ead0717cf9a73a6d1e779b23d149b53daf971169289ed2ed43a71e8d3b0"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1435ae15108b1cb6fffbcea2af3d468683b7afed0169ad718451f8db5d1aff6f"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c84132a54c750fda57729d1e2599bb598f5fa0344085dbde5003ba429a4798c0"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75f2568b4189dda1c567339b48cba4ac7384accb9c2a7ed655cd86b04055c795"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:11d3bcb7be35e7b1bba2c23beedac81ee893ac9871d0ba79effc7fc01167db6c"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:891cf9b48776b5c61c700b55a598621fdb7b1e301a550365571e9624f270c203"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:5f008525e02908b20e04707a4f704cd286d94718f48bb33edddc7d7b584dddc1"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:b06f0d3bf045158d2fb8837c5785fe9ff9b8c93358be64461a1089f5da983137"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:49919f8400b5e49e961f320c735388ee686a62327e773fa5b3ce6721f7e785ce"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:22908891a380d50738e1f978667536f6c6b526a2064156203d418f4856d6e86a"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-win32.whl", hash = "sha256:12d1a39aa6b8c6f6248bb54550efcc1c38ce0d8096a146638fd4738e42284448"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:65ed923f84a6844de5fd29726b888e58c62820e0769b76565480e1fdc3d062f8"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9a3267620866c9d17b959a84dd0bd2d45719b817245e49371ead79ed4f710d19"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6734e606355834f13445b6adc38b53c0fd45f1a56a9ba06c2058f86893ae8017"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f8303414c7b03f794347ad062c0516cee0e15f7a612abd0ce1e25caf6ceb47df"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aaf53a6cebad0eae578f062c7d462155eada9c172bd8c4d250b8c1d8eb7f916a"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3dc5b6a8ecfdc5748a7e429782598e4f17ef378e3e272eeb1340ea57c9109f41"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e1b25e3ad6c909f398df8921780d6a3d120d8c09466720226fc621605b6f92b1"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0ca564606d2caafb0abe6d1b5311c2649e8071eb241b2d64e75a0d0065107e62"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b82fab78e0b1329e183a65260581de4375f619167478dddab510c6c6fb04d9b6"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:bd7163182133c0c7701b25e604cf1611c0d87712e56e88e7ee5d72deab3e76b5"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:11d117e6c63e8f495412d37e7dc2e2fff09c34b2d09dbe2bee3c6229577818be"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:cf6511efa4801b9b38dc5546d7547d5b5c6ef4b081c60b23e4d941d0eba9cbeb"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:abc1185d79f47c0a7aaf7e2412a0eb2c03b724581139193d2d82b3ad8cbb00ac"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cb7b2ab0188829593b9de646545175547a70d9a6e2b63bf2cd87a0a391599324"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-win32.whl", hash = "sha256:c36bcbc0d5174a80d6cccf43a0ecaca44e81d25be4b7f90f0ed7bcfbb5a00909"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:cca4def576f47a09a943666b8f829606bcb17e2bc2d5911a46c8f8da45f56755"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0c95f12b74681e9ae127728f7e5409cbbef9cd914d5896ef238cc779b8152373"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fca62a8301b605b954ad2e9c3666f9d97f63872aa4efcae5492baca2056b74ab"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ac0aa6cd53ab9a31d397f8303f92c42f534693528fafbdb997c82bae6e477ad9"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c3af8e0f07399d3176b179f2e2634c3ce9c1301379a6b8c9c9aeecd481da494f"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a5fc78f9e3f501a1614a98f7c54d3969f3ad9bba8ba3d9b438c3bc5d047dd28"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:628c985afb2c7d27a4800bfb609e03985aaecb42f955049957814e0491d4006d"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:74db0052d985cf37fa111828d0dd230776ac99c740e1a758ad99094be4f1803d"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:1e8fcdd8f672a1c4fc8d0bd3a2b576b152d2a349782d1eb0f6b8e52e9954731d"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:04afa6387e2b282cf78ff3dbce20f0cc071c12dc8f685bd40960cc68644cfea6"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:dd5653e67b149503c68c4018bf07e42eeed6b4e956b24c00ccdf93ac79cdff84"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d2686f91611f9e17f4548dbf050e75b079bbc2a82be565832bc8ea9047b61c8c"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-win32.whl", hash = "sha256:4155b51ae05ed47199dc5b2a4e62abccb274cee6b01da5b895099b61b1982974"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-win_amd64.whl", hash = "sha256:322102cdf1ab682ecc7d9b1c5eed4ec59657a65e1c146a0da342b78f4112db23"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:e633940f28c1e913615fd624fcdd72fdba807bf53ea6925d6a588e84e1151531"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:3a06f32c9634a8705f4ca9946d667609f52cf130d5548881401f1eb2c39b1e2c"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7381c66e0561c5757ffe616af869b916c8b4e42b367ab29fedc98481d1e74e14"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3573d376454d956553c356df45bb824262c397c6e26ce43e8203c4c540ee0acb"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e89df2958e5159b811af9ff0f92614dabf4ff617c03a4c1c6ff53bf1c399e0e1"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:78cacd03e79d009d95635e7d6ff12c21eb89b894c354bd2b2ed0b4763373693b"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:de5695a6f1d8340b12a5d6d4484290ee74d61e467c39ff03b39e30df62cf83a0"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1c60b9c202d00052183c9be85e5eaf18a4ada0a47d188a83c8f5c5b23252f649"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:f645caaf0008bacf349875a974220f1f1da349c5dbe7c4ec93048cdc785a3326"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:ea9f9c6034ea2d93d9147818f17c2a0860d41b71c38b9ce4d55f21b6f9165a11"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:80d1543d58bd3d6c271b66abf454d437a438dff01c3e62fdbcd68f2a11310d4b"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:73dc03a6a7e30b7edc5b01b601e53e7fc924b04e1835e8e407c12c037e81adbd"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6f5c2e7bc8a4bf7c426599765b1bd33217ec84023033672c1e9a8b35eaeaaaf8"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-win32.whl", hash = "sha256:12a2b561af122e3d94cdb97fe6fb2bb2b82cef0cdca131646fdb940a1eda04f0"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:3160a0fd9754aab7d47f95a6b63ab355388d890163eb03b2d2b87ab0a30cfa59"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:38e812a197bf8e71a59fe55b757a84c1f946d0ac114acafaafaf21667a7e169e"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6baf0baf0d5d265fa7944feb9f7451cc316bfe30e8df1a61b1bb08577c554f31"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8f25e17ab3039b05f762b0a55ae0b3632b2e073d9c8fc88e89aca31a6198e88f"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3747443b6a904001473370d7810aa19c3a180ccd52a7157aacc264a5ac79265e"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b116502087ce8a6b7a5f1814568ccbd0e9f6cfd99948aa59b0e241dc57cf739f"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d16fd5252f883eb074ca55cb622bc0bee49b979ae4e8639fff6ca3ff44f9f854"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21fa558996782fc226b529fdd2ed7866c2c6ec91cee82735c98a197fae39f706"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6f6c7a8a57e9405cad7485f4c9d3172ae486cfef1344b5ddd8e5239582d7355e"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ac3775e3311661d4adace3697a52ac0bab17edd166087d493b52d4f4f553f9f0"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:10c93628d7497c81686e8e5e557aafa78f230cd9e77dd0c40032ef90c18f2230"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:6f4f4668e1831850ebcc2fd0b1cd11721947b6dc7c00bf1c6bd3c929ae14f2c7"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:0be65ccf618c1e7ac9b849c315cc2e8a8751d9cfdaa43027d4f6624bd587ab7e"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:53d0a3fa5f8af98a1e261de6a3943ca631c526635eb5817a87a59d9a57ebf48f"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-win32.whl", hash = "sha256:a04f86f41a8916fe45ac5024ec477f41f886b3c435da2d4e3d2709b22ab02af1"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:830d2948a5ec37c386d3170c483063798d7879037492540f10a475e3fd6f244b"}, - {file = "charset_normalizer-3.1.0-py3-none-any.whl", hash = "sha256:3d9098b479e78c85080c98e1e35ff40b4a31d8953102bb0fd7d1b6f8a2111a3d"}, +optional = false +python-versions = ">=3.7" +groups = ["main", "test"] +files = [ + {file = "charset_normalizer-3.4.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e824f1492727fa856dd6eda4f7cee25f8518a12f3c4a56a74e8095695089cf6d"}, + {file = "charset_normalizer-3.4.4-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4bd5d4137d500351a30687c2d3971758aac9a19208fc110ccb9d7188fbe709e8"}, + {file = "charset_normalizer-3.4.4-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:027f6de494925c0ab2a55eab46ae5129951638a49a34d87f4c3eda90f696b4ad"}, + {file = "charset_normalizer-3.4.4-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f820802628d2694cb7e56db99213f930856014862f3fd943d290ea8438d07ca8"}, + {file = "charset_normalizer-3.4.4-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:798d75d81754988d2565bff1b97ba5a44411867c0cf32b77a7e8f8d84796b10d"}, + {file = "charset_normalizer-3.4.4-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d1bb833febdff5c8927f922386db610b49db6e0d4f4ee29601d71e7c2694313"}, + {file = "charset_normalizer-3.4.4-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9cd98cdc06614a2f768d2b7286d66805f94c48cde050acdbbb7db2600ab3197e"}, + {file = "charset_normalizer-3.4.4-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:077fbb858e903c73f6c9db43374fd213b0b6a778106bc7032446a8e8b5b38b93"}, + {file = "charset_normalizer-3.4.4-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:244bfb999c71b35de57821b8ea746b24e863398194a4014e4c76adc2bbdfeff0"}, + {file = "charset_normalizer-3.4.4-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:64b55f9dce520635f018f907ff1b0df1fdc31f2795a922fb49dd14fbcdf48c84"}, + {file = "charset_normalizer-3.4.4-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:faa3a41b2b66b6e50f84ae4a68c64fcd0c44355741c6374813a800cd6695db9e"}, + {file = "charset_normalizer-3.4.4-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:6515f3182dbe4ea06ced2d9e8666d97b46ef4c75e326b79bb624110f122551db"}, + {file = "charset_normalizer-3.4.4-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:cc00f04ed596e9dc0da42ed17ac5e596c6ccba999ba6bd92b0e0aef2f170f2d6"}, + {file = "charset_normalizer-3.4.4-cp310-cp310-win32.whl", hash = "sha256:f34be2938726fc13801220747472850852fe6b1ea75869a048d6f896838c896f"}, + {file = "charset_normalizer-3.4.4-cp310-cp310-win_amd64.whl", hash = "sha256:a61900df84c667873b292c3de315a786dd8dac506704dea57bc957bd31e22c7d"}, + {file = "charset_normalizer-3.4.4-cp310-cp310-win_arm64.whl", hash = "sha256:cead0978fc57397645f12578bfd2d5ea9138ea0fac82b2f63f7f7c6877986a69"}, + {file = "charset_normalizer-3.4.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6e1fcf0720908f200cd21aa4e6750a48ff6ce4afe7ff5a79a90d5ed8a08296f8"}, + {file = "charset_normalizer-3.4.4-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5f819d5fe9234f9f82d75bdfa9aef3a3d72c4d24a6e57aeaebba32a704553aa0"}, + {file = "charset_normalizer-3.4.4-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a59cb51917aa591b1c4e6a43c132f0cdc3c76dbad6155df4e28ee626cc77a0a3"}, + {file = "charset_normalizer-3.4.4-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8ef3c867360f88ac904fd3f5e1f902f13307af9052646963ee08ff4f131adafc"}, + {file = "charset_normalizer-3.4.4-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d9e45d7faa48ee908174d8fe84854479ef838fc6a705c9315372eacbc2f02897"}, + {file = "charset_normalizer-3.4.4-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:840c25fb618a231545cbab0564a799f101b63b9901f2569faecd6b222ac72381"}, + {file = "charset_normalizer-3.4.4-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ca5862d5b3928c4940729dacc329aa9102900382fea192fc5e52eb69d6093815"}, + {file = "charset_normalizer-3.4.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d9c7f57c3d666a53421049053eaacdd14bbd0a528e2186fcb2e672effd053bb0"}, + {file = "charset_normalizer-3.4.4-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:277e970e750505ed74c832b4bf75dac7476262ee2a013f5574dd49075879e161"}, + {file = "charset_normalizer-3.4.4-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:31fd66405eaf47bb62e8cd575dc621c56c668f27d46a61d975a249930dd5e2a4"}, + {file = "charset_normalizer-3.4.4-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:0d3d8f15c07f86e9ff82319b3d9ef6f4bf907608f53fe9d92b28ea9ae3d1fd89"}, + {file = "charset_normalizer-3.4.4-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:9f7fcd74d410a36883701fafa2482a6af2ff5ba96b9a620e9e0721e28ead5569"}, + {file = "charset_normalizer-3.4.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ebf3e58c7ec8a8bed6d66a75d7fb37b55e5015b03ceae72a8e7c74495551e224"}, + {file = "charset_normalizer-3.4.4-cp311-cp311-win32.whl", hash = "sha256:eecbc200c7fd5ddb9a7f16c7decb07b566c29fa2161a16cf67b8d068bd21690a"}, + {file = "charset_normalizer-3.4.4-cp311-cp311-win_amd64.whl", hash = "sha256:5ae497466c7901d54b639cf42d5b8c1b6a4fead55215500d2f486d34db48d016"}, + {file = "charset_normalizer-3.4.4-cp311-cp311-win_arm64.whl", hash = "sha256:65e2befcd84bc6f37095f5961e68a6f077bf44946771354a28ad434c2cce0ae1"}, + {file = "charset_normalizer-3.4.4-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:0a98e6759f854bd25a58a73fa88833fba3b7c491169f86ce1180c948ab3fd394"}, + {file = "charset_normalizer-3.4.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b5b290ccc2a263e8d185130284f8501e3e36c5e02750fc6b6bdeb2e9e96f1e25"}, + {file = "charset_normalizer-3.4.4-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:74bb723680f9f7a6234dcf67aea57e708ec1fbdf5699fb91dfd6f511b0a320ef"}, + {file = "charset_normalizer-3.4.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f1e34719c6ed0b92f418c7c780480b26b5d9c50349e9a9af7d76bf757530350d"}, + {file = "charset_normalizer-3.4.4-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2437418e20515acec67d86e12bf70056a33abdacb5cb1655042f6538d6b085a8"}, + {file = "charset_normalizer-3.4.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:11d694519d7f29d6cd09f6ac70028dba10f92f6cdd059096db198c283794ac86"}, + {file = "charset_normalizer-3.4.4-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ac1c4a689edcc530fc9d9aa11f5774b9e2f33f9a0c6a57864e90908f5208d30a"}, + {file = "charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:21d142cc6c0ec30d2efee5068ca36c128a30b0f2c53c1c07bd78cb6bc1d3be5f"}, + {file = "charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:5dbe56a36425d26d6cfb40ce79c314a2e4dd6211d51d6d2191c00bed34f354cc"}, + {file = "charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:5bfbb1b9acf3334612667b61bd3002196fe2a1eb4dd74d247e0f2a4d50ec9bbf"}, + {file = "charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:d055ec1e26e441f6187acf818b73564e6e6282709e9bcb5b63f5b23068356a15"}, + {file = "charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:af2d8c67d8e573d6de5bc30cdb27e9b95e49115cd9baad5ddbd1a6207aaa82a9"}, + {file = "charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:780236ac706e66881f3b7f2f32dfe90507a09e67d1d454c762cf642e6e1586e0"}, + {file = "charset_normalizer-3.4.4-cp312-cp312-win32.whl", hash = "sha256:5833d2c39d8896e4e19b689ffc198f08ea58116bee26dea51e362ecc7cd3ed26"}, + {file = "charset_normalizer-3.4.4-cp312-cp312-win_amd64.whl", hash = "sha256:a79cfe37875f822425b89a82333404539ae63dbdddf97f84dcbc3d339aae9525"}, + {file = "charset_normalizer-3.4.4-cp312-cp312-win_arm64.whl", hash = "sha256:376bec83a63b8021bb5c8ea75e21c4ccb86e7e45ca4eb81146091b56599b80c3"}, + {file = "charset_normalizer-3.4.4-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:e1f185f86a6f3403aa2420e815904c67b2f9ebc443f045edd0de921108345794"}, + {file = "charset_normalizer-3.4.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b39f987ae8ccdf0d2642338faf2abb1862340facc796048b604ef14919e55ed"}, + {file = "charset_normalizer-3.4.4-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3162d5d8ce1bb98dd51af660f2121c55d0fa541b46dff7bb9b9f86ea1d87de72"}, + {file = "charset_normalizer-3.4.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:81d5eb2a312700f4ecaa977a8235b634ce853200e828fbadf3a9c50bab278328"}, + {file = "charset_normalizer-3.4.4-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5bd2293095d766545ec1a8f612559f6b40abc0eb18bb2f5d1171872d34036ede"}, + {file = "charset_normalizer-3.4.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a8a8b89589086a25749f471e6a900d3f662d1d3b6e2e59dcecf787b1cc3a1894"}, + {file = "charset_normalizer-3.4.4-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:bc7637e2f80d8530ee4a78e878bce464f70087ce73cf7c1caf142416923b98f1"}, + {file = "charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f8bf04158c6b607d747e93949aa60618b61312fe647a6369f88ce2ff16043490"}, + {file = "charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:554af85e960429cf30784dd47447d5125aaa3b99a6f0683589dbd27e2f45da44"}, + {file = "charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:74018750915ee7ad843a774364e13a3db91682f26142baddf775342c3f5b1133"}, + {file = "charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:c0463276121fdee9c49b98908b3a89c39be45d86d1dbaa22957e38f6321d4ce3"}, + {file = "charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:362d61fd13843997c1c446760ef36f240cf81d3ebf74ac62652aebaf7838561e"}, + {file = "charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9a26f18905b8dd5d685d6d07b0cdf98a79f3c7a918906af7cc143ea2e164c8bc"}, + {file = "charset_normalizer-3.4.4-cp313-cp313-win32.whl", hash = "sha256:9b35f4c90079ff2e2edc5b26c0c77925e5d2d255c42c74fdb70fb49b172726ac"}, + {file = "charset_normalizer-3.4.4-cp313-cp313-win_amd64.whl", hash = "sha256:b435cba5f4f750aa6c0a0d92c541fb79f69a387c91e61f1795227e4ed9cece14"}, + {file = "charset_normalizer-3.4.4-cp313-cp313-win_arm64.whl", hash = "sha256:542d2cee80be6f80247095cc36c418f7bddd14f4a6de45af91dfad36d817bba2"}, + {file = "charset_normalizer-3.4.4-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:da3326d9e65ef63a817ecbcc0df6e94463713b754fe293eaa03da99befb9a5bd"}, + {file = "charset_normalizer-3.4.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8af65f14dc14a79b924524b1e7fffe304517b2bff5a58bf64f30b98bbc5079eb"}, + {file = "charset_normalizer-3.4.4-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:74664978bb272435107de04e36db5a9735e78232b85b77d45cfb38f758efd33e"}, + {file = "charset_normalizer-3.4.4-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:752944c7ffbfdd10c074dc58ec2d5a8a4cd9493b314d367c14d24c17684ddd14"}, + {file = "charset_normalizer-3.4.4-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d1f13550535ad8cff21b8d757a3257963e951d96e20ec82ab44bc64aeb62a191"}, + {file = "charset_normalizer-3.4.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ecaae4149d99b1c9e7b88bb03e3221956f68fd6d50be2ef061b2381b61d20838"}, + {file = "charset_normalizer-3.4.4-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cb6254dc36b47a990e59e1068afacdcd02958bdcce30bb50cc1700a8b9d624a6"}, + {file = "charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c8ae8a0f02f57a6e61203a31428fa1d677cbe50c93622b4149d5c0f319c1d19e"}, + {file = "charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:47cc91b2f4dd2833fddaedd2893006b0106129d4b94fdb6af1f4ce5a9965577c"}, + {file = "charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:82004af6c302b5d3ab2cfc4cc5f29db16123b1a8417f2e25f9066f91d4411090"}, + {file = "charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2b7d8f6c26245217bd2ad053761201e9f9680f8ce52f0fcd8d0755aeae5b2152"}, + {file = "charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:799a7a5e4fb2d5898c60b640fd4981d6a25f1c11790935a44ce38c54e985f828"}, + {file = "charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:99ae2cffebb06e6c22bdc25801d7b30f503cc87dbd283479e7b606f70aff57ec"}, + {file = "charset_normalizer-3.4.4-cp314-cp314-win32.whl", hash = "sha256:f9d332f8c2a2fcbffe1378594431458ddbef721c1769d78e2cbc06280d8155f9"}, + {file = "charset_normalizer-3.4.4-cp314-cp314-win_amd64.whl", hash = "sha256:8a6562c3700cce886c5be75ade4a5db4214fda19fede41d9792d100288d8f94c"}, + {file = "charset_normalizer-3.4.4-cp314-cp314-win_arm64.whl", hash = "sha256:de00632ca48df9daf77a2c65a484531649261ec9f25489917f09e455cb09ddb2"}, + {file = "charset_normalizer-3.4.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:ce8a0633f41a967713a59c4139d29110c07e826d131a316b50ce11b1d79b4f84"}, + {file = "charset_normalizer-3.4.4-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eaabd426fe94daf8fd157c32e571c85cb12e66692f15516a83a03264b08d06c3"}, + {file = "charset_normalizer-3.4.4-cp38-cp38-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c4ef880e27901b6cc782f1b95f82da9313c0eb95c3af699103088fa0ac3ce9ac"}, + {file = "charset_normalizer-3.4.4-cp38-cp38-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2aaba3b0819274cc41757a1da876f810a3e4d7b6eb25699253a4effef9e8e4af"}, + {file = "charset_normalizer-3.4.4-cp38-cp38-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:778d2e08eda00f4256d7f672ca9fef386071c9202f5e4607920b86d7803387f2"}, + {file = "charset_normalizer-3.4.4-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f155a433c2ec037d4e8df17d18922c3a0d9b3232a396690f17175d2946f0218d"}, + {file = "charset_normalizer-3.4.4-cp38-cp38-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a8bf8d0f749c5757af2142fe7903a9df1d2e8aa3841559b2bad34b08d0e2bcf3"}, + {file = "charset_normalizer-3.4.4-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:194f08cbb32dc406d6e1aea671a68be0823673db2832b38405deba2fb0d88f63"}, + {file = "charset_normalizer-3.4.4-cp38-cp38-musllinux_1_2_armv7l.whl", hash = "sha256:6aee717dcfead04c6eb1ce3bd29ac1e22663cdea57f943c87d1eab9a025438d7"}, + {file = "charset_normalizer-3.4.4-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:cd4b7ca9984e5e7985c12bc60a6f173f3c958eae74f3ef6624bb6b26e2abbae4"}, + {file = "charset_normalizer-3.4.4-cp38-cp38-musllinux_1_2_riscv64.whl", hash = "sha256:b7cf1017d601aa35e6bb650b6ad28652c9cd78ee6caff19f3c28d03e1c80acbf"}, + {file = "charset_normalizer-3.4.4-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:e912091979546adf63357d7e2ccff9b44f026c075aeaf25a52d0e95ad2281074"}, + {file = "charset_normalizer-3.4.4-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:5cb4d72eea50c8868f5288b7f7f33ed276118325c1dfd3957089f6b519e1382a"}, + {file = "charset_normalizer-3.4.4-cp38-cp38-win32.whl", hash = "sha256:837c2ce8c5a65a2035be9b3569c684358dfbf109fd3b6969630a87535495ceaa"}, + {file = "charset_normalizer-3.4.4-cp38-cp38-win_amd64.whl", hash = "sha256:44c2a8734b333e0578090c4cd6b16f275e07aa6614ca8715e6c038e865e70576"}, + {file = "charset_normalizer-3.4.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:a9768c477b9d7bd54bc0c86dbaebdec6f03306675526c9927c0e8a04e8f94af9"}, + {file = "charset_normalizer-3.4.4-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1bee1e43c28aa63cb16e5c14e582580546b08e535299b8b6158a7c9c768a1f3d"}, + {file = "charset_normalizer-3.4.4-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:fd44c878ea55ba351104cb93cc85e74916eb8fa440ca7903e57575e97394f608"}, + {file = "charset_normalizer-3.4.4-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0f04b14ffe5fdc8c4933862d8306109a2c51e0704acfa35d51598eb45a1e89fc"}, + {file = "charset_normalizer-3.4.4-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:cd09d08005f958f370f539f186d10aec3377d55b9eeb0d796025d4886119d76e"}, + {file = "charset_normalizer-3.4.4-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4fe7859a4e3e8457458e2ff592f15ccb02f3da787fcd31e0183879c3ad4692a1"}, + {file = "charset_normalizer-3.4.4-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fa09f53c465e532f4d3db095e0c55b615f010ad81803d383195b6b5ca6cbf5f3"}, + {file = "charset_normalizer-3.4.4-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:7fa17817dc5625de8a027cb8b26d9fefa3ea28c8253929b8d6649e705d2835b6"}, + {file = "charset_normalizer-3.4.4-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:5947809c8a2417be3267efc979c47d76a079758166f7d43ef5ae8e9f92751f88"}, + {file = "charset_normalizer-3.4.4-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:4902828217069c3c5c71094537a8e623f5d097858ac6ca8252f7b4d10b7560f1"}, + {file = "charset_normalizer-3.4.4-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:7c308f7e26e4363d79df40ca5b2be1c6ba9f02bdbccfed5abddb7859a6ce72cf"}, + {file = "charset_normalizer-3.4.4-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:2c9d3c380143a1fedbff95a312aa798578371eb29da42106a29019368a475318"}, + {file = "charset_normalizer-3.4.4-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:cb01158d8b88ee68f15949894ccc6712278243d95f344770fa7593fa2d94410c"}, + {file = "charset_normalizer-3.4.4-cp39-cp39-win32.whl", hash = "sha256:2677acec1a2f8ef614c6888b5b4ae4060cc184174a938ed4e8ef690e15d3e505"}, + {file = "charset_normalizer-3.4.4-cp39-cp39-win_amd64.whl", hash = "sha256:f8e160feb2aed042cd657a72acc0b481212ed28b1b9a95c0cee1621b524e1966"}, + {file = "charset_normalizer-3.4.4-cp39-cp39-win_arm64.whl", hash = "sha256:b5d84d37db046c5ca74ee7bb47dd6cbc13f80665fdde3e8040bdd3fb015ecb50"}, + {file = "charset_normalizer-3.4.4-py3-none-any.whl", hash = "sha256:7a32c560861a02ff789ad905a2fe94e3f840803362c84fecf1851cb4cf3dc37f"}, + {file = "charset_normalizer-3.4.4.tar.gz", hash = "sha256:94537985111c35f28720e43603b8e7b43a6ecfb2ce1d3058bbe955b73404e21a"}, ] [[package]] name = "click" -version = "8.1.3" +version = "8.1.8" description = "Composable command line interface toolkit" -category = "main" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ - {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, - {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, + {file = "click-8.1.8-py3-none-any.whl", hash = "sha256:63c132bbbed01578a06712a2d1f497bb62d9c1c0d329b7903a866228027263b2"}, + {file = "click-8.1.8.tar.gz", hash = "sha256:ed53c9d8990d83c2a27deae68e4ee337473f6330c040a31d4225c9574d16096a"}, ] [package.dependencies] colorama = {version = "*", markers = "platform_system == \"Windows\""} -importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} [[package]] name = "colorama" version = "0.4.6" description = "Cross-platform colored terminal text." -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +groups = ["main", "test"] files = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, @@ -181,109 +342,195 @@ files = [ [[package]] name = "coverage" -version = "7.2.5" +version = "7.2.7" description = "Code coverage measurement for Python" -category = "dev" optional = false python-versions = ">=3.7" +groups = ["test"] files = [ - {file = "coverage-7.2.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:883123d0bbe1c136f76b56276074b0c79b5817dd4238097ffa64ac67257f4b6c"}, - {file = "coverage-7.2.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d2fbc2a127e857d2f8898aaabcc34c37771bf78a4d5e17d3e1f5c30cd0cbc62a"}, - {file = "coverage-7.2.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5f3671662dc4b422b15776cdca89c041a6349b4864a43aa2350b6b0b03bbcc7f"}, - {file = "coverage-7.2.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:780551e47d62095e088f251f5db428473c26db7829884323e56d9c0c3118791a"}, - {file = "coverage-7.2.5-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:066b44897c493e0dcbc9e6a6d9f8bbb6607ef82367cf6810d387c09f0cd4fe9a"}, - {file = "coverage-7.2.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:b9a4ee55174b04f6af539218f9f8083140f61a46eabcaa4234f3c2a452c4ed11"}, - {file = "coverage-7.2.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:706ec567267c96717ab9363904d846ec009a48d5f832140b6ad08aad3791b1f5"}, - {file = "coverage-7.2.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:ae453f655640157d76209f42c62c64c4d4f2c7f97256d3567e3b439bd5c9b06c"}, - {file = "coverage-7.2.5-cp310-cp310-win32.whl", hash = "sha256:f81c9b4bd8aa747d417407a7f6f0b1469a43b36a85748145e144ac4e8d303cb5"}, - {file = "coverage-7.2.5-cp310-cp310-win_amd64.whl", hash = "sha256:dc945064a8783b86fcce9a0a705abd7db2117d95e340df8a4333f00be5efb64c"}, - {file = "coverage-7.2.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:40cc0f91c6cde033da493227797be2826cbf8f388eaa36a0271a97a332bfd7ce"}, - {file = "coverage-7.2.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a66e055254a26c82aead7ff420d9fa8dc2da10c82679ea850d8feebf11074d88"}, - {file = "coverage-7.2.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c10fbc8a64aa0f3ed136b0b086b6b577bc64d67d5581acd7cc129af52654384e"}, - {file = "coverage-7.2.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9a22cbb5ede6fade0482111fa7f01115ff04039795d7092ed0db43522431b4f2"}, - {file = "coverage-7.2.5-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:292300f76440651529b8ceec283a9370532f4ecba9ad67d120617021bb5ef139"}, - {file = "coverage-7.2.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:7ff8f3fb38233035028dbc93715551d81eadc110199e14bbbfa01c5c4a43f8d8"}, - {file = "coverage-7.2.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:a08c7401d0b24e8c2982f4e307124b671c6736d40d1c39e09d7a8687bddf83ed"}, - {file = "coverage-7.2.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:ef9659d1cda9ce9ac9585c045aaa1e59223b143f2407db0eaee0b61a4f266fb6"}, - {file = "coverage-7.2.5-cp311-cp311-win32.whl", hash = "sha256:30dcaf05adfa69c2a7b9f7dfd9f60bc8e36b282d7ed25c308ef9e114de7fc23b"}, - {file = "coverage-7.2.5-cp311-cp311-win_amd64.whl", hash = "sha256:97072cc90f1009386c8a5b7de9d4fc1a9f91ba5ef2146c55c1f005e7b5c5e068"}, - {file = "coverage-7.2.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:bebea5f5ed41f618797ce3ffb4606c64a5de92e9c3f26d26c2e0aae292f015c1"}, - {file = "coverage-7.2.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:828189fcdda99aae0d6bf718ea766b2e715eabc1868670a0a07bf8404bf58c33"}, - {file = "coverage-7.2.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6e8a95f243d01ba572341c52f89f3acb98a3b6d1d5d830efba86033dd3687ade"}, - {file = "coverage-7.2.5-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e8834e5f17d89e05697c3c043d3e58a8b19682bf365048837383abfe39adaed5"}, - {file = "coverage-7.2.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:d1f25ee9de21a39b3a8516f2c5feb8de248f17da7eead089c2e04aa097936b47"}, - {file = "coverage-7.2.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:1637253b11a18f453e34013c665d8bf15904c9e3c44fbda34c643fbdc9d452cd"}, - {file = "coverage-7.2.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8e575a59315a91ccd00c7757127f6b2488c2f914096077c745c2f1ba5b8c0969"}, - {file = "coverage-7.2.5-cp37-cp37m-win32.whl", hash = "sha256:509ecd8334c380000d259dc66feb191dd0a93b21f2453faa75f7f9cdcefc0718"}, - {file = "coverage-7.2.5-cp37-cp37m-win_amd64.whl", hash = "sha256:12580845917b1e59f8a1c2ffa6af6d0908cb39220f3019e36c110c943dc875b0"}, - {file = "coverage-7.2.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b5016e331b75310610c2cf955d9f58a9749943ed5f7b8cfc0bb89c6134ab0a84"}, - {file = "coverage-7.2.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:373ea34dca98f2fdb3e5cb33d83b6d801007a8074f992b80311fc589d3e6b790"}, - {file = "coverage-7.2.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a063aad9f7b4c9f9da7b2550eae0a582ffc7623dca1c925e50c3fbde7a579771"}, - {file = "coverage-7.2.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:38c0a497a000d50491055805313ed83ddba069353d102ece8aef5d11b5faf045"}, - {file = "coverage-7.2.5-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a2b3b05e22a77bb0ae1a3125126a4e08535961c946b62f30985535ed40e26614"}, - {file = "coverage-7.2.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:0342a28617e63ad15d96dca0f7ae9479a37b7d8a295f749c14f3436ea59fdcb3"}, - {file = "coverage-7.2.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:cf97ed82ca986e5c637ea286ba2793c85325b30f869bf64d3009ccc1a31ae3fd"}, - {file = "coverage-7.2.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:c2c41c1b1866b670573657d584de413df701f482574bad7e28214a2362cb1fd1"}, - {file = "coverage-7.2.5-cp38-cp38-win32.whl", hash = "sha256:10b15394c13544fce02382360cab54e51a9e0fd1bd61ae9ce012c0d1e103c813"}, - {file = "coverage-7.2.5-cp38-cp38-win_amd64.whl", hash = "sha256:a0b273fe6dc655b110e8dc89b8ec7f1a778d78c9fd9b4bda7c384c8906072212"}, - {file = "coverage-7.2.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5c587f52c81211d4530fa6857884d37f514bcf9453bdeee0ff93eaaf906a5c1b"}, - {file = "coverage-7.2.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4436cc9ba5414c2c998eaedee5343f49c02ca93b21769c5fdfa4f9d799e84200"}, - {file = "coverage-7.2.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6599bf92f33ab041e36e06d25890afbdf12078aacfe1f1d08c713906e49a3fe5"}, - {file = "coverage-7.2.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:857abe2fa6a4973f8663e039ead8d22215d31db613ace76e4a98f52ec919068e"}, - {file = "coverage-7.2.5-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f6f5cab2d7f0c12f8187a376cc6582c477d2df91d63f75341307fcdcb5d60303"}, - {file = "coverage-7.2.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:aa387bd7489f3e1787ff82068b295bcaafbf6f79c3dad3cbc82ef88ce3f48ad3"}, - {file = "coverage-7.2.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:156192e5fd3dbbcb11cd777cc469cf010a294f4c736a2b2c891c77618cb1379a"}, - {file = "coverage-7.2.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:bd3b4b8175c1db502adf209d06136c000df4d245105c8839e9d0be71c94aefe1"}, - {file = "coverage-7.2.5-cp39-cp39-win32.whl", hash = "sha256:ddc5a54edb653e9e215f75de377354e2455376f416c4378e1d43b08ec50acc31"}, - {file = "coverage-7.2.5-cp39-cp39-win_amd64.whl", hash = "sha256:338aa9d9883aaaad53695cb14ccdeb36d4060485bb9388446330bef9c361c252"}, - {file = "coverage-7.2.5-pp37.pp38.pp39-none-any.whl", hash = "sha256:8877d9b437b35a85c18e3c6499b23674684bf690f5d96c1006a1ef61f9fdf0f3"}, - {file = "coverage-7.2.5.tar.gz", hash = "sha256:f99ef080288f09ffc687423b8d60978cf3a465d3f404a18d1a05474bd8575a47"}, -] - -[package.extras] -toml = ["tomli"] + {file = "coverage-7.2.7-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d39b5b4f2a66ccae8b7263ac3c8170994b65266797fb96cbbfd3fb5b23921db8"}, + {file = "coverage-7.2.7-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6d040ef7c9859bb11dfeb056ff5b3872436e3b5e401817d87a31e1750b9ae2fb"}, + {file = "coverage-7.2.7-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ba90a9563ba44a72fda2e85302c3abc71c5589cea608ca16c22b9804262aaeb6"}, + {file = "coverage-7.2.7-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e7d9405291c6928619403db1d10bd07888888ec1abcbd9748fdaa971d7d661b2"}, + {file = "coverage-7.2.7-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:31563e97dae5598556600466ad9beea39fb04e0229e61c12eaa206e0aa202063"}, + {file = "coverage-7.2.7-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:ebba1cd308ef115925421d3e6a586e655ca5a77b5bf41e02eb0e4562a111f2d1"}, + {file = "coverage-7.2.7-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:cb017fd1b2603ef59e374ba2063f593abe0fc45f2ad9abdde5b4d83bd922a353"}, + {file = "coverage-7.2.7-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d62a5c7dad11015c66fbb9d881bc4caa5b12f16292f857842d9d1871595f4495"}, + {file = "coverage-7.2.7-cp310-cp310-win32.whl", hash = "sha256:ee57190f24fba796e36bb6d3aa8a8783c643d8fa9760c89f7a98ab5455fbf818"}, + {file = "coverage-7.2.7-cp310-cp310-win_amd64.whl", hash = "sha256:f75f7168ab25dd93110c8a8117a22450c19976afbc44234cbf71481094c1b850"}, + {file = "coverage-7.2.7-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:06a9a2be0b5b576c3f18f1a241f0473575c4a26021b52b2a85263a00f034d51f"}, + {file = "coverage-7.2.7-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5baa06420f837184130752b7c5ea0808762083bf3487b5038d68b012e5937dbe"}, + {file = "coverage-7.2.7-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fdec9e8cbf13a5bf63290fc6013d216a4c7232efb51548594ca3631a7f13c3a3"}, + {file = "coverage-7.2.7-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:52edc1a60c0d34afa421c9c37078817b2e67a392cab17d97283b64c5833f427f"}, + {file = "coverage-7.2.7-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:63426706118b7f5cf6bb6c895dc215d8a418d5952544042c8a2d9fe87fcf09cb"}, + {file = "coverage-7.2.7-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:afb17f84d56068a7c29f5fa37bfd38d5aba69e3304af08ee94da8ed5b0865833"}, + {file = "coverage-7.2.7-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:48c19d2159d433ccc99e729ceae7d5293fbffa0bdb94952d3579983d1c8c9d97"}, + {file = "coverage-7.2.7-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0e1f928eaf5469c11e886fe0885ad2bf1ec606434e79842a879277895a50942a"}, + {file = "coverage-7.2.7-cp311-cp311-win32.whl", hash = "sha256:33d6d3ea29d5b3a1a632b3c4e4f4ecae24ef170b0b9ee493883f2df10039959a"}, + {file = "coverage-7.2.7-cp311-cp311-win_amd64.whl", hash = "sha256:5b7540161790b2f28143191f5f8ec02fb132660ff175b7747b95dcb77ac26562"}, + {file = "coverage-7.2.7-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:f2f67fe12b22cd130d34d0ef79206061bfb5eda52feb6ce0dba0644e20a03cf4"}, + {file = "coverage-7.2.7-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a342242fe22407f3c17f4b499276a02b01e80f861f1682ad1d95b04018e0c0d4"}, + {file = "coverage-7.2.7-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:171717c7cb6b453aebac9a2ef603699da237f341b38eebfee9be75d27dc38e01"}, + {file = "coverage-7.2.7-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:49969a9f7ffa086d973d91cec8d2e31080436ef0fb4a359cae927e742abfaaa6"}, + {file = "coverage-7.2.7-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b46517c02ccd08092f4fa99f24c3b83d8f92f739b4657b0f146246a0ca6a831d"}, + {file = "coverage-7.2.7-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:a3d33a6b3eae87ceaefa91ffdc130b5e8536182cd6dfdbfc1aa56b46ff8c86de"}, + {file = "coverage-7.2.7-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:976b9c42fb2a43ebf304fa7d4a310e5f16cc99992f33eced91ef6f908bd8f33d"}, + {file = "coverage-7.2.7-cp312-cp312-win32.whl", hash = "sha256:8de8bb0e5ad103888d65abef8bca41ab93721647590a3f740100cd65c3b00511"}, + {file = "coverage-7.2.7-cp312-cp312-win_amd64.whl", hash = "sha256:9e31cb64d7de6b6f09702bb27c02d1904b3aebfca610c12772452c4e6c21a0d3"}, + {file = "coverage-7.2.7-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:58c2ccc2f00ecb51253cbe5d8d7122a34590fac9646a960d1430d5b15321d95f"}, + {file = "coverage-7.2.7-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d22656368f0e6189e24722214ed8d66b8022db19d182927b9a248a2a8a2f67eb"}, + {file = "coverage-7.2.7-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a895fcc7b15c3fc72beb43cdcbdf0ddb7d2ebc959edac9cef390b0d14f39f8a9"}, + {file = "coverage-7.2.7-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e84606b74eb7de6ff581a7915e2dab7a28a0517fbe1c9239eb227e1354064dcd"}, + {file = "coverage-7.2.7-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:0a5f9e1dbd7fbe30196578ca36f3fba75376fb99888c395c5880b355e2875f8a"}, + {file = "coverage-7.2.7-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:419bfd2caae268623dd469eff96d510a920c90928b60f2073d79f8fe2bbc5959"}, + {file = "coverage-7.2.7-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:2aee274c46590717f38ae5e4650988d1af340fe06167546cc32fe2f58ed05b02"}, + {file = "coverage-7.2.7-cp37-cp37m-win32.whl", hash = "sha256:61b9a528fb348373c433e8966535074b802c7a5d7f23c4f421e6c6e2f1697a6f"}, + {file = "coverage-7.2.7-cp37-cp37m-win_amd64.whl", hash = "sha256:b1c546aca0ca4d028901d825015dc8e4d56aac4b541877690eb76490f1dc8ed0"}, + {file = "coverage-7.2.7-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:54b896376ab563bd38453cecb813c295cf347cf5906e8b41d340b0321a5433e5"}, + {file = "coverage-7.2.7-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:3d376df58cc111dc8e21e3b6e24606b5bb5dee6024f46a5abca99124b2229ef5"}, + {file = "coverage-7.2.7-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5e330fc79bd7207e46c7d7fd2bb4af2963f5f635703925543a70b99574b0fea9"}, + {file = "coverage-7.2.7-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e9d683426464e4a252bf70c3498756055016f99ddaec3774bf368e76bbe02b6"}, + {file = "coverage-7.2.7-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d13c64ee2d33eccf7437961b6ea7ad8673e2be040b4f7fd4fd4d4d28d9ccb1e"}, + {file = "coverage-7.2.7-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:b7aa5f8a41217360e600da646004f878250a0d6738bcdc11a0a39928d7dc2050"}, + {file = "coverage-7.2.7-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:8fa03bce9bfbeeef9f3b160a8bed39a221d82308b4152b27d82d8daa7041fee5"}, + {file = "coverage-7.2.7-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:245167dd26180ab4c91d5e1496a30be4cd721a5cf2abf52974f965f10f11419f"}, + {file = "coverage-7.2.7-cp38-cp38-win32.whl", hash = "sha256:d2c2db7fd82e9b72937969bceac4d6ca89660db0a0967614ce2481e81a0b771e"}, + {file = "coverage-7.2.7-cp38-cp38-win_amd64.whl", hash = "sha256:2e07b54284e381531c87f785f613b833569c14ecacdcb85d56b25c4622c16c3c"}, + {file = "coverage-7.2.7-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:537891ae8ce59ef63d0123f7ac9e2ae0fc8b72c7ccbe5296fec45fd68967b6c9"}, + {file = "coverage-7.2.7-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:06fb182e69f33f6cd1d39a6c597294cff3143554b64b9825d1dc69d18cc2fff2"}, + {file = "coverage-7.2.7-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:201e7389591af40950a6480bd9edfa8ed04346ff80002cec1a66cac4549c1ad7"}, + {file = "coverage-7.2.7-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f6951407391b639504e3b3be51b7ba5f3528adbf1a8ac3302b687ecababf929e"}, + {file = "coverage-7.2.7-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f48351d66575f535669306aa7d6d6f71bc43372473b54a832222803eb956fd1"}, + {file = "coverage-7.2.7-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b29019c76039dc3c0fd815c41392a044ce555d9bcdd38b0fb60fb4cd8e475ba9"}, + {file = "coverage-7.2.7-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:81c13a1fc7468c40f13420732805a4c38a105d89848b7c10af65a90beff25250"}, + {file = "coverage-7.2.7-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:975d70ab7e3c80a3fe86001d8751f6778905ec723f5b110aed1e450da9d4b7f2"}, + {file = "coverage-7.2.7-cp39-cp39-win32.whl", hash = "sha256:7ee7d9d4822c8acc74a5e26c50604dff824710bc8de424904c0982e25c39c6cb"}, + {file = "coverage-7.2.7-cp39-cp39-win_amd64.whl", hash = "sha256:eb393e5ebc85245347950143969b241d08b52b88a3dc39479822e073a1a8eb27"}, + {file = "coverage-7.2.7-pp37.pp38.pp39-none-any.whl", hash = "sha256:b7b4c971f05e6ae490fef852c218b0e79d4e52f79ef0c8475566584a8fb3e01d"}, + {file = "coverage-7.2.7.tar.gz", hash = "sha256:924d94291ca674905fe9481f12294eb11f2d3d3fd1adb20314ba89e94f44ed59"}, +] + +[package.extras] +toml = ["tomli ; python_full_version <= \"3.11.0a6\""] + +[[package]] +name = "cryptography" +version = "46.0.5" +description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." +optional = false +python-versions = "!=3.9.0,!=3.9.1,>=3.8" +groups = ["main"] +markers = "python_version >= \"3.10\"" +files = [ + {file = "cryptography-46.0.5-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:351695ada9ea9618b3500b490ad54c739860883df6c1f555e088eaf25b1bbaad"}, + {file = "cryptography-46.0.5-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c18ff11e86df2e28854939acde2d003f7984f721eba450b56a200ad90eeb0e6b"}, + {file = "cryptography-46.0.5-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4d7e3d356b8cd4ea5aff04f129d5f66ebdc7b6f8eae802b93739ed520c47c79b"}, + {file = "cryptography-46.0.5-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:50bfb6925eff619c9c023b967d5b77a54e04256c4281b0e21336a130cd7fc263"}, + {file = "cryptography-46.0.5-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:803812e111e75d1aa73690d2facc295eaefd4439be1023fefc4995eaea2af90d"}, + {file = "cryptography-46.0.5-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:3ee190460e2fbe447175cda91b88b84ae8322a104fc27766ad09428754a618ed"}, + {file = "cryptography-46.0.5-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:f145bba11b878005c496e93e257c1e88f154d278d2638e6450d17e0f31e558d2"}, + {file = "cryptography-46.0.5-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:e9251e3be159d1020c4030bd2e5f84d6a43fe54b6c19c12f51cde9542a2817b2"}, + {file = "cryptography-46.0.5-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:47fb8a66058b80e509c47118ef8a75d14c455e81ac369050f20ba0d23e77fee0"}, + {file = "cryptography-46.0.5-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:4c3341037c136030cb46e4b1e17b7418ea4cbd9dd207e4a6f3b2b24e0d4ac731"}, + {file = "cryptography-46.0.5-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:890bcb4abd5a2d3f852196437129eb3667d62630333aacc13dfd470fad3aaa82"}, + {file = "cryptography-46.0.5-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:80a8d7bfdf38f87ca30a5391c0c9ce4ed2926918e017c29ddf643d0ed2778ea1"}, + {file = "cryptography-46.0.5-cp311-abi3-win32.whl", hash = "sha256:60ee7e19e95104d4c03871d7d7dfb3d22ef8a9b9c6778c94e1c8fcc8365afd48"}, + {file = "cryptography-46.0.5-cp311-abi3-win_amd64.whl", hash = "sha256:38946c54b16c885c72c4f59846be9743d699eee2b69b6988e0a00a01f46a61a4"}, + {file = "cryptography-46.0.5-cp314-cp314t-macosx_10_9_universal2.whl", hash = "sha256:94a76daa32eb78d61339aff7952ea819b1734b46f73646a07decb40e5b3448e2"}, + {file = "cryptography-46.0.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5be7bf2fb40769e05739dd0046e7b26f9d4670badc7b032d6ce4db64dddc0678"}, + {file = "cryptography-46.0.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fe346b143ff9685e40192a4960938545c699054ba11d4f9029f94751e3f71d87"}, + {file = "cryptography-46.0.5-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:c69fd885df7d089548a42d5ec05be26050ebcd2283d89b3d30676eb32ff87dee"}, + {file = "cryptography-46.0.5-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:8293f3dea7fc929ef7240796ba231413afa7b68ce38fd21da2995549f5961981"}, + {file = "cryptography-46.0.5-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:1abfdb89b41c3be0365328a410baa9df3ff8a9110fb75e7b52e66803ddabc9a9"}, + {file = "cryptography-46.0.5-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:d66e421495fdb797610a08f43b05269e0a5ea7f5e652a89bfd5a7d3c1dee3648"}, + {file = "cryptography-46.0.5-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:4e817a8920bfbcff8940ecfd60f23d01836408242b30f1a708d93198393a80b4"}, + {file = "cryptography-46.0.5-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:68f68d13f2e1cb95163fa3b4db4bf9a159a418f5f6e7242564fc75fcae667fd0"}, + {file = "cryptography-46.0.5-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:a3d1fae9863299076f05cb8a778c467578262fae09f9dc0ee9b12eb4268ce663"}, + {file = "cryptography-46.0.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c4143987a42a2397f2fc3b4d7e3a7d313fbe684f67ff443999e803dd75a76826"}, + {file = "cryptography-46.0.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:7d731d4b107030987fd61a7f8ab512b25b53cef8f233a97379ede116f30eb67d"}, + {file = "cryptography-46.0.5-cp314-cp314t-win32.whl", hash = "sha256:c3bcce8521d785d510b2aad26ae2c966092b7daa8f45dd8f44734a104dc0bc1a"}, + {file = "cryptography-46.0.5-cp314-cp314t-win_amd64.whl", hash = "sha256:4d8ae8659ab18c65ced284993c2265910f6c9e650189d4e3f68445ef82a810e4"}, + {file = "cryptography-46.0.5-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:4108d4c09fbbf2789d0c926eb4152ae1760d5a2d97612b92d508d96c861e4d31"}, + {file = "cryptography-46.0.5-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7d1f30a86d2757199cb2d56e48cce14deddf1f9c95f1ef1b64ee91ea43fe2e18"}, + {file = "cryptography-46.0.5-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:039917b0dc418bb9f6edce8a906572d69e74bd330b0b3fea4f79dab7f8ddd235"}, + {file = "cryptography-46.0.5-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:ba2a27ff02f48193fc4daeadf8ad2590516fa3d0adeeb34336b96f7fa64c1e3a"}, + {file = "cryptography-46.0.5-cp38-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:61aa400dce22cb001a98014f647dc21cda08f7915ceb95df0c9eaf84b4b6af76"}, + {file = "cryptography-46.0.5-cp38-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:3ce58ba46e1bc2aac4f7d9290223cead56743fa6ab94a5d53292ffaac6a91614"}, + {file = "cryptography-46.0.5-cp38-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:420d0e909050490d04359e7fdb5ed7e667ca5c3c402b809ae2563d7e66a92229"}, + {file = "cryptography-46.0.5-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:582f5fcd2afa31622f317f80426a027f30dc792e9c80ffee87b993200ea115f1"}, + {file = "cryptography-46.0.5-cp38-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:bfd56bb4b37ed4f330b82402f6f435845a5f5648edf1ad497da51a8452d5d62d"}, + {file = "cryptography-46.0.5-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:a3d507bb6a513ca96ba84443226af944b0f7f47dcc9a399d110cd6146481d24c"}, + {file = "cryptography-46.0.5-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:9f16fbdf4da055efb21c22d81b89f155f02ba420558db21288b3d0035bafd5f4"}, + {file = "cryptography-46.0.5-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:ced80795227d70549a411a4ab66e8ce307899fad2220ce5ab2f296e687eacde9"}, + {file = "cryptography-46.0.5-cp38-abi3-win32.whl", hash = "sha256:02f547fce831f5096c9a567fd41bc12ca8f11df260959ecc7c3202555cc47a72"}, + {file = "cryptography-46.0.5-cp38-abi3-win_amd64.whl", hash = "sha256:556e106ee01aa13484ce9b0239bca667be5004efb0aabbed28d353df86445595"}, + {file = "cryptography-46.0.5-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:3b4995dc971c9fb83c25aa44cf45f02ba86f71ee600d81091c2f0cbae116b06c"}, + {file = "cryptography-46.0.5-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:bc84e875994c3b445871ea7181d424588171efec3e185dced958dad9e001950a"}, + {file = "cryptography-46.0.5-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:2ae6971afd6246710480e3f15824ed3029a60fc16991db250034efd0b9fb4356"}, + {file = "cryptography-46.0.5-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:d861ee9e76ace6cf36a6a89b959ec08e7bc2493ee39d07ffe5acb23ef46d27da"}, + {file = "cryptography-46.0.5-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:2b7a67c9cd56372f3249b39699f2ad479f6991e62ea15800973b956f4b73e257"}, + {file = "cryptography-46.0.5-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:8456928655f856c6e1533ff59d5be76578a7157224dbd9ce6872f25055ab9ab7"}, + {file = "cryptography-46.0.5.tar.gz", hash = "sha256:abace499247268e3757271b2f1e244b36b06f8515cf27c4d49468fc9eb16e93d"}, +] + +[package.dependencies] +cffi = {version = ">=2.0.0", markers = "python_full_version >= \"3.9.0\" and platform_python_implementation != \"PyPy\""} +typing-extensions = {version = ">=4.13.2", markers = "python_full_version < \"3.11.0\""} + +[package.extras] +docs = ["sphinx (>=5.3.0)", "sphinx-inline-tabs", "sphinx-rtd-theme (>=3.0.0)"] +docstest = ["pyenchant (>=3)", "readme-renderer (>=30.0)", "sphinxcontrib-spelling (>=7.3.1)"] +nox = ["nox[uv] (>=2024.4.15)"] +pep8test = ["check-sdist", "click (>=8.0.1)", "mypy (>=1.14)", "ruff (>=0.11.11)"] +sdist = ["build (>=1.0.0)"] +ssh = ["bcrypt (>=3.1.5)"] +test = ["certifi (>=2024)", "cryptography-vectors (==46.0.5)", "pretend (>=0.7)", "pytest (>=7.4.0)", "pytest-benchmark (>=4.0)", "pytest-cov (>=2.10.1)", "pytest-xdist (>=3.5.0)"] +test-randomorder = ["pytest-randomly"] [[package]] name = "dunamai" -version = "1.16.1" +version = "1.26.0" description = "Dynamic version generation" -category = "dev" optional = false -python-versions = ">=3.5,<4.0" +python-versions = ">=3.5" +groups = ["executable"] files = [ - {file = "dunamai-1.16.1-py3-none-any.whl", hash = "sha256:b9f169183147f6f1d3a5b3d913ffdd67247d90948006e205cbc499fe98d45554"}, - {file = "dunamai-1.16.1.tar.gz", hash = "sha256:4f3bc2c5b0f9d83fa9c90b943100273bb087167c90a0519ac66e9e2e0d2a8210"}, + {file = "dunamai-1.26.0-py3-none-any.whl", hash = "sha256:f584edf0fda0d308cce0961f807bc90a8fe3d9ff4d62f94e72eca7b43f0ed5f6"}, + {file = "dunamai-1.26.0.tar.gz", hash = "sha256:5396ac43aa20ed059040034e9f9798c7464cf4334c6fc3da3732e29273a2f97d"}, ] [package.dependencies] -importlib-metadata = {version = ">=1.6.0", markers = "python_version < \"3.8\""} packaging = ">=20.9" [[package]] name = "exceptiongroup" -version = "1.1.1" +version = "1.3.0" description = "Backport of PEP 654 (exception groups)" -category = "dev" optional = false python-versions = ">=3.7" +groups = ["main", "test"] files = [ - {file = "exceptiongroup-1.1.1-py3-none-any.whl", hash = "sha256:232c37c63e4f682982c8b6459f33a8981039e5fb8756b2074364e5055c498c9e"}, - {file = "exceptiongroup-1.1.1.tar.gz", hash = "sha256:d484c3090ba2889ae2928419117447a14daf3c1231d5e30d0aae34f354f01785"}, + {file = "exceptiongroup-1.3.0-py3-none-any.whl", hash = "sha256:4d111e6e0c13d0644cad6ddaa7ed0261a0b36971f6d23e7ec9b4b9097da78a10"}, + {file = "exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88"}, ] +markers = {main = "python_version == \"3.10\"", test = "python_version < \"3.11\""} + +[package.dependencies] +typing-extensions = {version = ">=4.6.0", markers = "python_version < \"3.13\""} [package.extras] test = ["pytest (>=6)"] [[package]] name = "gitdb" -version = "4.0.10" +version = "4.0.12" description = "Git Object Database" -category = "main" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ - {file = "gitdb-4.0.10-py3-none-any.whl", hash = "sha256:c286cf298426064079ed96a9e4a9d39e7f3e9bf15ba60701e95f5492f28415c7"}, - {file = "gitdb-4.0.10.tar.gz", hash = "sha256:6eb990b69df4e15bad899ea868dc46572c3f75339735663b81de79b06f17eb9a"}, + {file = "gitdb-4.0.12-py3-none-any.whl", hash = "sha256:67073e15955400952c6565cc3e707c554a4eea2e428946f7a4c162fab9bd9bcf"}, + {file = "gitdb-4.0.12.tar.gz", hash = "sha256:5ef71f855d191a3326fcfbc0d5da835f26b13fbcba60c32c21091c349ffdb571"}, ] [package.dependencies] @@ -291,142 +538,304 @@ smmap = ">=3.0.1,<6" [[package]] name = "gitpython" -version = "3.1.31" +version = "3.1.45" description = "GitPython is a Python library used to interact with Git repositories" -category = "main" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ - {file = "GitPython-3.1.31-py3-none-any.whl", hash = "sha256:f04893614f6aa713a60cbbe1e6a97403ef633103cdd0ef5eb6efe0deb98dbe8d"}, - {file = "GitPython-3.1.31.tar.gz", hash = "sha256:8ce3bcf69adfdf7c7d503e78fd3b1c492af782d58893b650adb2ac8912ddd573"}, + {file = "gitpython-3.1.45-py3-none-any.whl", hash = "sha256:8908cb2e02fb3b93b7eb0f2827125cb699869470432cc885f019b8fd0fccff77"}, + {file = "gitpython-3.1.45.tar.gz", hash = "sha256:85b0ee964ceddf211c41b9f27a49086010a190fd8132a24e21f362a4b36a791c"}, ] [package.dependencies] gitdb = ">=4.0.1,<5" -typing-extensions = {version = ">=3.7.4.3", markers = "python_version < \"3.8\""} +typing-extensions = {version = ">=3.10.0.2", markers = "python_version < \"3.10\""} + +[package.extras] +doc = ["sphinx (>=7.1.2,<7.2)", "sphinx-autodoc-typehints", "sphinx_rtd_theme"] +test = ["coverage[toml]", "ddt (>=1.1.1,!=1.4.3)", "mock ; python_version < \"3.8\"", "mypy", "pre-commit", "pytest (>=7.3.1)", "pytest-cov", "pytest-instafail", "pytest-mock", "pytest-sugar", "typing-extensions ; python_version < \"3.11\""] + +[[package]] +name = "h11" +version = "0.16.0" +description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" +optional = false +python-versions = ">=3.8" +groups = ["main"] +markers = "python_version >= \"3.10\"" +files = [ + {file = "h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86"}, + {file = "h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1"}, +] + +[[package]] +name = "httpcore" +version = "1.0.9" +description = "A minimal low-level HTTP client." +optional = false +python-versions = ">=3.8" +groups = ["main"] +markers = "python_version >= \"3.10\"" +files = [ + {file = "httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55"}, + {file = "httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8"}, +] + +[package.dependencies] +certifi = "*" +h11 = ">=0.16" + +[package.extras] +asyncio = ["anyio (>=4.0,<5.0)"] +http2 = ["h2 (>=3,<5)"] +socks = ["socksio (==1.*)"] +trio = ["trio (>=0.22.0,<1.0)"] [[package]] -name = "halo" -version = "0.0.31" -description = "Beautiful terminal spinners in Python" -category = "main" +name = "httpx" +version = "0.28.1" +description = "The next generation HTTP client." optional = false -python-versions = ">=3.4" +python-versions = ">=3.8" +groups = ["main"] +markers = "python_version >= \"3.10\"" files = [ - {file = "halo-0.0.31-py2-none-any.whl", hash = "sha256:5350488fb7d2aa7c31a1344120cee67a872901ce8858f60da7946cef96c208ab"}, - {file = "halo-0.0.31.tar.gz", hash = "sha256:7b67a3521ee91d53b7152d4ee3452811e1d2a6321975137762eb3d70063cc9d6"}, + {file = "httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad"}, + {file = "httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc"}, ] [package.dependencies] -colorama = ">=0.3.9" -log-symbols = ">=0.0.14" -six = ">=1.12.0" -spinners = ">=0.0.24" -termcolor = ">=1.1.0" +anyio = "*" +certifi = "*" +httpcore = "==1.*" +idna = "*" [package.extras] -ipython = ["IPython (==5.7.0)", "ipywidgets (==7.1.0)"] +brotli = ["brotli ; platform_python_implementation == \"CPython\"", "brotlicffi ; platform_python_implementation != \"CPython\""] +cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"] +http2 = ["h2 (>=3,<5)"] +socks = ["socksio (==1.*)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "httpx-sse" +version = "0.4.3" +description = "Consume Server-Sent Event (SSE) messages with HTTPX." +optional = false +python-versions = ">=3.9" +groups = ["main"] +markers = "python_version >= \"3.10\"" +files = [ + {file = "httpx_sse-0.4.3-py3-none-any.whl", hash = "sha256:0ac1c9fe3c0afad2e0ebb25a934a59f4c7823b60792691f779fad2c5568830fc"}, + {file = "httpx_sse-0.4.3.tar.gz", hash = "sha256:9b1ed0127459a66014aec3c56bebd93da3c1bc8bb6618c8082039a44889a755d"}, +] [[package]] name = "idna" -version = "3.4" +version = "3.11" description = "Internationalized Domain Names in Applications (IDNA)" -category = "main" optional = false -python-versions = ">=3.5" +python-versions = ">=3.8" +groups = ["main", "test"] files = [ - {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, - {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, + {file = "idna-3.11-py3-none-any.whl", hash = "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea"}, + {file = "idna-3.11.tar.gz", hash = "sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902"}, ] +[package.extras] +all = ["flake8 (>=7.1.1)", "mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2)"] + [[package]] name = "importlib-metadata" -version = "6.6.0" +version = "8.7.0" description = "Read metadata from Python packages" -category = "main" optional = false -python-versions = ">=3.7" +python-versions = ">=3.9" +groups = ["executable"] +markers = "python_version == \"3.9\"" files = [ - {file = "importlib_metadata-6.6.0-py3-none-any.whl", hash = "sha256:43dd286a2cd8995d5eaef7fee2066340423b818ed3fd70adf0bad5f1fac53fed"}, - {file = "importlib_metadata-6.6.0.tar.gz", hash = "sha256:92501cdf9cc66ebd3e612f1b4f0c0765dfa42f0fa38ffb319b6bd84dd675d705"}, + {file = "importlib_metadata-8.7.0-py3-none-any.whl", hash = "sha256:e5dd1551894c77868a30651cef00984d50e1002d06942a7101d34870c5f02afd"}, + {file = "importlib_metadata-8.7.0.tar.gz", hash = "sha256:d13b81ad223b890aa16c5471f2ac3056cf76c5f10f82d6f9292f0b415f389000"}, ] [package.dependencies] -typing-extensions = {version = ">=3.6.4", markers = "python_version < \"3.8\""} -zipp = ">=0.5" +zipp = ">=3.20" [package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\""] +cover = ["pytest-cov"] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +enabler = ["pytest-enabler (>=2.2)"] perf = ["ipython"] -testing = ["flake8 (<5)", "flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)"] +test = ["flufl.flake8", "importlib_resources (>=1.3) ; python_version < \"3.9\"", "jaraco.test (>=5.4)", "packaging", "pyfakefs", "pytest (>=6,!=8.1.*)", "pytest-perf (>=0.9.2)"] +type = ["pytest-mypy"] [[package]] name = "iniconfig" -version = "2.0.0" +version = "2.1.0" description = "brain-dead simple config-ini parsing" -category = "dev" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" +groups = ["test"] files = [ - {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, - {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, + {file = "iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760"}, + {file = "iniconfig-2.1.0.tar.gz", hash = "sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7"}, ] [[package]] -name = "log-symbols" -version = "0.0.14" -description = "Colored symbols for various log levels for Python" -category = "main" +name = "jsonschema" +version = "4.25.1" +description = "An implementation of JSON Schema validation for Python" optional = false -python-versions = "*" +python-versions = ">=3.9" +groups = ["main"] +markers = "python_version >= \"3.10\"" +files = [ + {file = "jsonschema-4.25.1-py3-none-any.whl", hash = "sha256:3fba0169e345c7175110351d456342c364814cfcf3b964ba4587f22915230a63"}, + {file = "jsonschema-4.25.1.tar.gz", hash = "sha256:e4a9655ce0da0c0b67a085847e00a3a51449e1157f4f75e9fb5aa545e122eb85"}, +] + +[package.dependencies] +attrs = ">=22.2.0" +jsonschema-specifications = ">=2023.03.6" +referencing = ">=0.28.4" +rpds-py = ">=0.7.1" + +[package.extras] +format = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3987", "uri-template", "webcolors (>=1.11)"] +format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "rfc3987-syntax (>=1.1.0)", "uri-template", "webcolors (>=24.6.0)"] + +[[package]] +name = "jsonschema-specifications" +version = "2025.9.1" +description = "The JSON Schema meta-schemas and vocabularies, exposed as a Registry" +optional = false +python-versions = ">=3.9" +groups = ["main"] +markers = "python_version >= \"3.10\"" files = [ - {file = "log_symbols-0.0.14-py3-none-any.whl", hash = "sha256:4952106ff8b605ab7d5081dd2c7e6ca7374584eff7086f499c06edd1ce56dcca"}, - {file = "log_symbols-0.0.14.tar.gz", hash = "sha256:cf0bbc6fe1a8e53f0d174a716bc625c4f87043cc21eb55dd8a740cfe22680556"}, + {file = "jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe"}, + {file = "jsonschema_specifications-2025.9.1.tar.gz", hash = "sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d"}, ] [package.dependencies] -colorama = ">=0.3.9" +referencing = ">=0.31.0" [[package]] name = "macholib" -version = "1.16.2" +version = "1.16.3" description = "Mach-O header analysis and editing" -category = "dev" optional = false python-versions = "*" +groups = ["executable"] +markers = "python_version < \"3.15\" and sys_platform == \"darwin\"" files = [ - {file = "macholib-1.16.2-py2.py3-none-any.whl", hash = "sha256:44c40f2cd7d6726af8fa6fe22549178d3a4dfecc35a9cd15ea916d9c83a688e0"}, - {file = "macholib-1.16.2.tar.gz", hash = "sha256:557bbfa1bb255c20e9abafe7ed6cd8046b48d9525db2f9b77d3122a63a2a8bf8"}, + {file = "macholib-1.16.3-py2.py3-none-any.whl", hash = "sha256:0e315d7583d38b8c77e815b1ecbdbf504a8258d8b3e17b61165c6feb60d18f2c"}, + {file = "macholib-1.16.3.tar.gz", hash = "sha256:07ae9e15e8e4cd9a788013d81f5908b3609aa76f9b1421bae9c4d7606ec86a30"}, ] [package.dependencies] altgraph = ">=0.17" +[[package]] +name = "markdown-it-py" +version = "3.0.0" +description = "Python port of markdown-it. Markdown parsing, done right!" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "markdown-it-py-3.0.0.tar.gz", hash = "sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb"}, + {file = "markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1"}, +] + +[package.dependencies] +mdurl = ">=0.1,<1.0" + +[package.extras] +benchmarking = ["psutil", "pytest", "pytest-benchmark"] +code-style = ["pre-commit (>=3.0,<4.0)"] +compare = ["commonmark (>=0.9,<1.0)", "markdown (>=3.4,<4.0)", "mistletoe (>=1.0,<2.0)", "mistune (>=2.0,<3.0)", "panflute (>=2.3,<3.0)"] +linkify = ["linkify-it-py (>=1,<3)"] +plugins = ["mdit-py-plugins"] +profiling = ["gprof2dot"] +rtd = ["jupyter_sphinx", "mdit-py-plugins", "myst-parser", "pyyaml", "sphinx", "sphinx-copybutton", "sphinx-design", "sphinx_book_theme"] +testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] + [[package]] name = "marshmallow" -version = "3.8.0" +version = "3.26.2" description = "A lightweight library for converting complex datatypes to and from native Python datatypes." -category = "main" optional = false -python-versions = ">=3.5" +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "marshmallow-3.26.2-py3-none-any.whl", hash = "sha256:013fa8a3c4c276c24d26d84ce934dc964e2aa794345a0f8c7e5a7191482c8a73"}, + {file = "marshmallow-3.26.2.tar.gz", hash = "sha256:bbe2adb5a03e6e3571b573f42527c6fe926e17467833660bebd11593ab8dfd57"}, +] + +[package.dependencies] +packaging = ">=17.0" + +[package.extras] +dev = ["marshmallow[tests]", "pre-commit (>=3.5,<5.0)", "tox"] +docs = ["autodocsumm (==0.2.14)", "furo (==2024.8.6)", "sphinx (==8.1.3)", "sphinx-copybutton (==0.5.2)", "sphinx-issues (==5.0.0)", "sphinxext-opengraph (==0.9.1)"] +tests = ["pytest", "simplejson"] + +[[package]] +name = "mcp" +version = "1.26.0" +description = "Model Context Protocol SDK" +optional = false +python-versions = ">=3.10" +groups = ["main"] +markers = "python_version >= \"3.10\"" files = [ - {file = "marshmallow-3.8.0-py2.py3-none-any.whl", hash = "sha256:2272273505f1644580fbc66c6b220cc78f893eb31f1ecde2af98ad28011e9811"}, - {file = "marshmallow-3.8.0.tar.gz", hash = "sha256:47911dd7c641a27160f0df5fd0fe94667160ffe97f70a42c3cc18388d86098cc"}, + {file = "mcp-1.26.0-py3-none-any.whl", hash = "sha256:904a21c33c25aa98ddbeb47273033c435e595bbacfdb177f4bd87f6dceebe1ca"}, + {file = "mcp-1.26.0.tar.gz", hash = "sha256:db6e2ef491eecc1a0d93711a76f28dec2e05999f93afd48795da1c1137142c66"}, ] +[package.dependencies] +anyio = ">=4.5" +httpx = ">=0.27.1" +httpx-sse = ">=0.4" +jsonschema = ">=4.20.0" +pydantic = ">=2.11.0,<3.0.0" +pydantic-settings = ">=2.5.2" +pyjwt = {version = ">=2.10.1", extras = ["crypto"]} +python-multipart = ">=0.0.9" +pywin32 = {version = ">=310", markers = "sys_platform == \"win32\""} +sse-starlette = ">=1.6.1" +starlette = ">=0.27" +typing-extensions = ">=4.9.0" +typing-inspection = ">=0.4.1" +uvicorn = {version = ">=0.31.1", markers = "sys_platform != \"emscripten\""} + [package.extras] -dev = ["flake8 (==3.8.3)", "flake8-bugbear (==20.1.4)", "mypy (==0.782)", "pre-commit (>=2.4,<3.0)", "pytest", "pytz", "simplejson", "tox"] -docs = ["alabaster (==0.7.12)", "autodocsumm (==0.2.0)", "sphinx (==3.2.1)", "sphinx-issues (==1.2.0)", "sphinx-version-warning (==1.1.2)"] -lint = ["flake8 (==3.8.3)", "flake8-bugbear (==20.1.4)", "mypy (==0.782)", "pre-commit (>=2.4,<3.0)"] -tests = ["pytest", "pytz", "simplejson"] +cli = ["python-dotenv (>=1.0.0)", "typer (>=0.16.0)"] +rich = ["rich (>=13.9.4)"] +ws = ["websockets (>=15.0.1)"] + +[[package]] +name = "mdurl" +version = "0.1.2" +description = "Markdown URL utilities" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"}, + {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, +] [[package]] name = "mock" version = "4.0.3" description = "Rolling backport of unittest.mock for all Pythons" -category = "dev" optional = false python-versions = ">=3.6" +groups = ["test"] files = [ {file = "mock-4.0.3-py3-none-any.whl", hash = "sha256:122fcb64ee37cfad5b3f48d7a7d51875d7031aaf3d8be7c42e2bee25044eee62"}, {file = "mock-4.0.3.tar.gz", hash = "sha256:7d3fbbde18228f4ff2f1f119a45cdffa458b4c0dee32eb4d2bb2f82554bac7bc"}, @@ -439,137 +848,396 @@ test = ["pytest (<5.4)", "pytest-cov"] [[package]] name = "packaging" -version = "23.1" +version = "25.0" description = "Core utilities for Python packages" -category = "dev" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" +groups = ["main", "executable", "test"] files = [ - {file = "packaging-23.1-py3-none-any.whl", hash = "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61"}, - {file = "packaging-23.1.tar.gz", hash = "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"}, + {file = "packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484"}, + {file = "packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f"}, ] [[package]] -name = "pathspec" -version = "0.8.1" -description = "Utility library for gitignore style pattern matching of file paths." -category = "main" +name = "patch-ng" +version = "1.19.0" +description = "Library to parse and apply unified diffs." optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +python-versions = ">=3.6" +groups = ["main"] files = [ - {file = "pathspec-0.8.1-py2.py3-none-any.whl", hash = "sha256:aa0cb481c4041bf52ffa7b0d8fa6cd3e88a2ca4879c533c9153882ee2556790d"}, - {file = "pathspec-0.8.1.tar.gz", hash = "sha256:86379d6b86d75816baba717e64b1a3a3469deb93bb76d613c9ce79edc5cb68fd"}, + {file = "patch-ng-1.19.0.tar.gz", hash = "sha256:27484792f4ac1c15fe2f3e4cecf74bb9833d33b75c715b71d199f7e1e7d1f786"}, ] +[[package]] +name = "pathvalidate" +version = "3.3.1" +description = "pathvalidate is a Python library to sanitize/validate a string such as filenames/file-paths/etc." +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "pathvalidate-3.3.1-py3-none-any.whl", hash = "sha256:5263baab691f8e1af96092fa5137ee17df5bdfbd6cff1fcac4d6ef4bc2e1735f"}, + {file = "pathvalidate-3.3.1.tar.gz", hash = "sha256:b18c07212bfead624345bb8e1d6141cdcf15a39736994ea0b94035ad2b1ba177"}, +] + +[package.extras] +docs = ["Sphinx (>=2.4)", "sphinx_rtd_theme (>=1.2.2)", "urllib3 (<2)"] +readme = ["path (>=13,<18)", "readmemaker (>=1.2.0)"] +test = ["Faker (>=1.0.8)", "allpairspy (>=2)", "click (>=6.2)", "pytest (>=6.0.1)", "pytest-md-report (>=0.6.2)"] + [[package]] name = "pefile" -version = "2023.2.7" +version = "2024.8.26" description = "Python PE parsing module" -category = "dev" optional = false python-versions = ">=3.6.0" +groups = ["executable"] +markers = "python_version < \"3.15\" and sys_platform == \"win32\"" files = [ - {file = "pefile-2023.2.7-py3-none-any.whl", hash = "sha256:da185cd2af68c08a6cd4481f7325ed600a88f6a813bad9dea07ab3ef73d8d8d6"}, - {file = "pefile-2023.2.7.tar.gz", hash = "sha256:82e6114004b3d6911c77c3953e3838654b04511b8b66e8583db70c65998017dc"}, + {file = "pefile-2024.8.26-py3-none-any.whl", hash = "sha256:76f8b485dcd3b1bb8166f1128d395fa3d87af26360c2358fb75b80019b957c6f"}, + {file = "pefile-2024.8.26.tar.gz", hash = "sha256:3ff6c5d8b43e8c37bb6e6dd5085658d658a7a0bdcd20b6a07b1fcfc1c4e9d632"}, ] [[package]] name = "pluggy" -version = "1.0.0" +version = "1.6.0" description = "plugin and hook calling mechanisms for python" -category = "dev" optional = false -python-versions = ">=3.6" +python-versions = ">=3.9" +groups = ["test"] files = [ - {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, - {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, + {file = "pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746"}, + {file = "pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["coverage", "pytest", "pytest-benchmark"] + +[[package]] +name = "pycparser" +version = "3.0" +description = "C parser in Python" +optional = false +python-versions = ">=3.10" +groups = ["main"] +markers = "python_version >= \"3.10\" and platform_python_implementation != \"PyPy\" and implementation_name != \"PyPy\"" +files = [ + {file = "pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992"}, + {file = "pycparser-3.0.tar.gz", hash = "sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29"}, +] + +[[package]] +name = "pydantic" +version = "2.12.3" +description = "Data validation using Python type hints" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "pydantic-2.12.3-py3-none-any.whl", hash = "sha256:6986454a854bc3bc6e5443e1369e06a3a456af9d339eda45510f517d9ea5c6bf"}, + {file = "pydantic-2.12.3.tar.gz", hash = "sha256:1da1c82b0fc140bb0103bc1441ffe062154c8d38491189751ee00fd8ca65ce74"}, ] [package.dependencies] -importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} +annotated-types = ">=0.6.0" +pydantic-core = "2.41.4" +typing-extensions = ">=4.14.1" +typing-inspection = ">=0.4.2" [package.extras] -dev = ["pre-commit", "tox"] -testing = ["pytest", "pytest-benchmark"] +email = ["email-validator (>=2.0.0)"] +timezone = ["tzdata ; python_version >= \"3.9\" and platform_system == \"Windows\""] + +[[package]] +name = "pydantic-core" +version = "2.41.4" +description = "Core functionality for Pydantic validation and serialization" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "pydantic_core-2.41.4-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:2442d9a4d38f3411f22eb9dd0912b7cbf4b7d5b6c92c4173b75d3e1ccd84e36e"}, + {file = "pydantic_core-2.41.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:30a9876226dda131a741afeab2702e2d127209bde3c65a2b8133f428bc5d006b"}, + {file = "pydantic_core-2.41.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d55bbac04711e2980645af68b97d445cdbcce70e5216de444a6c4b6943ebcccd"}, + {file = "pydantic_core-2.41.4-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e1d778fb7849a42d0ee5927ab0f7453bf9f85eef8887a546ec87db5ddb178945"}, + {file = "pydantic_core-2.41.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1b65077a4693a98b90ec5ad8f203ad65802a1b9b6d4a7e48066925a7e1606706"}, + {file = "pydantic_core-2.41.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:62637c769dee16eddb7686bf421be48dfc2fae93832c25e25bc7242e698361ba"}, + {file = "pydantic_core-2.41.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2dfe3aa529c8f501babf6e502936b9e8d4698502b2cfab41e17a028d91b1ac7b"}, + {file = "pydantic_core-2.41.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ca2322da745bf2eeb581fc9ea3bbb31147702163ccbcbf12a3bb630e4bf05e1d"}, + {file = "pydantic_core-2.41.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e8cd3577c796be7231dcf80badcf2e0835a46665eaafd8ace124d886bab4d700"}, + {file = "pydantic_core-2.41.4-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:1cae8851e174c83633f0833e90636832857297900133705ee158cf79d40f03e6"}, + {file = "pydantic_core-2.41.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a26d950449aae348afe1ac8be5525a00ae4235309b729ad4d3399623125b43c9"}, + {file = "pydantic_core-2.41.4-cp310-cp310-win32.whl", hash = "sha256:0cf2a1f599efe57fa0051312774280ee0f650e11152325e41dfd3018ef2c1b57"}, + {file = "pydantic_core-2.41.4-cp310-cp310-win_amd64.whl", hash = "sha256:a8c2e340d7e454dc3340d3d2e8f23558ebe78c98aa8f68851b04dcb7bc37abdc"}, + {file = "pydantic_core-2.41.4-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:28ff11666443a1a8cf2a044d6a545ebffa8382b5f7973f22c36109205e65dc80"}, + {file = "pydantic_core-2.41.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:61760c3925d4633290292bad462e0f737b840508b4f722247d8729684f6539ae"}, + {file = "pydantic_core-2.41.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eae547b7315d055b0de2ec3965643b0ab82ad0106a7ffd29615ee9f266a02827"}, + {file = "pydantic_core-2.41.4-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ef9ee5471edd58d1fcce1c80ffc8783a650e3e3a193fe90d52e43bb4d87bff1f"}, + {file = "pydantic_core-2.41.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:15dd504af121caaf2c95cb90c0ebf71603c53de98305621b94da0f967e572def"}, + {file = "pydantic_core-2.41.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3a926768ea49a8af4d36abd6a8968b8790f7f76dd7cbd5a4c180db2b4ac9a3a2"}, + {file = "pydantic_core-2.41.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6916b9b7d134bff5440098a4deb80e4cb623e68974a87883299de9124126c2a8"}, + {file = "pydantic_core-2.41.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5cf90535979089df02e6f17ffd076f07237efa55b7343d98760bde8743c4b265"}, + {file = "pydantic_core-2.41.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:7533c76fa647fade2d7ec75ac5cc079ab3f34879626dae5689b27790a6cf5a5c"}, + {file = "pydantic_core-2.41.4-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:37e516bca9264cbf29612539801ca3cd5d1be465f940417b002905e6ed79d38a"}, + {file = "pydantic_core-2.41.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0c19cb355224037c83642429b8ce261ae108e1c5fbf5c028bac63c77b0f8646e"}, + {file = "pydantic_core-2.41.4-cp311-cp311-win32.whl", hash = "sha256:09c2a60e55b357284b5f31f5ab275ba9f7f70b7525e18a132ec1f9160b4f1f03"}, + {file = "pydantic_core-2.41.4-cp311-cp311-win_amd64.whl", hash = "sha256:711156b6afb5cb1cb7c14a2cc2c4a8b4c717b69046f13c6b332d8a0a8f41ca3e"}, + {file = "pydantic_core-2.41.4-cp311-cp311-win_arm64.whl", hash = "sha256:6cb9cf7e761f4f8a8589a45e49ed3c0d92d1d696a45a6feaee8c904b26efc2db"}, + {file = "pydantic_core-2.41.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:ab06d77e053d660a6faaf04894446df7b0a7e7aba70c2797465a0a1af00fc887"}, + {file = "pydantic_core-2.41.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c53ff33e603a9c1179a9364b0a24694f183717b2e0da2b5ad43c316c956901b2"}, + {file = "pydantic_core-2.41.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:304c54176af2c143bd181d82e77c15c41cbacea8872a2225dd37e6544dce9999"}, + {file = "pydantic_core-2.41.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:025ba34a4cf4fb32f917d5d188ab5e702223d3ba603be4d8aca2f82bede432a4"}, + {file = "pydantic_core-2.41.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b9f5f30c402ed58f90c70e12eff65547d3ab74685ffe8283c719e6bead8ef53f"}, + {file = "pydantic_core-2.41.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dd96e5d15385d301733113bcaa324c8bcf111275b7675a9c6e88bfb19fc05e3b"}, + {file = "pydantic_core-2.41.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98f348cbb44fae6e9653c1055db7e29de67ea6a9ca03a5fa2c2e11a47cff0e47"}, + {file = "pydantic_core-2.41.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ec22626a2d14620a83ca583c6f5a4080fa3155282718b6055c2ea48d3ef35970"}, + {file = "pydantic_core-2.41.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:3a95d4590b1f1a43bf33ca6d647b990a88f4a3824a8c4572c708f0b45a5290ed"}, + {file = "pydantic_core-2.41.4-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:f9672ab4d398e1b602feadcffcdd3af44d5f5e6ddc15bc7d15d376d47e8e19f8"}, + {file = "pydantic_core-2.41.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:84d8854db5f55fead3b579f04bda9a36461dab0730c5d570e1526483e7bb8431"}, + {file = "pydantic_core-2.41.4-cp312-cp312-win32.whl", hash = "sha256:9be1c01adb2ecc4e464392c36d17f97e9110fbbc906bcbe1c943b5b87a74aabd"}, + {file = "pydantic_core-2.41.4-cp312-cp312-win_amd64.whl", hash = "sha256:d682cf1d22bab22a5be08539dca3d1593488a99998f9f412137bc323179067ff"}, + {file = "pydantic_core-2.41.4-cp312-cp312-win_arm64.whl", hash = "sha256:833eebfd75a26d17470b58768c1834dfc90141b7afc6eb0429c21fc5a21dcfb8"}, + {file = "pydantic_core-2.41.4-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:85e050ad9e5f6fe1004eec65c914332e52f429bc0ae12d6fa2092407a462c746"}, + {file = "pydantic_core-2.41.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e7393f1d64792763a48924ba31d1e44c2cfbc05e3b1c2c9abb4ceeadd912cced"}, + {file = "pydantic_core-2.41.4-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94dab0940b0d1fb28bcab847adf887c66a27a40291eedf0b473be58761c9799a"}, + {file = "pydantic_core-2.41.4-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:de7c42f897e689ee6f9e93c4bec72b99ae3b32a2ade1c7e4798e690ff5246e02"}, + {file = "pydantic_core-2.41.4-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:664b3199193262277b8b3cd1e754fb07f2c6023289c815a1e1e8fb415cb247b1"}, + {file = "pydantic_core-2.41.4-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d95b253b88f7d308b1c0b417c4624f44553ba4762816f94e6986819b9c273fb2"}, + {file = "pydantic_core-2.41.4-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a1351f5bbdbbabc689727cb91649a00cb9ee7203e0a6e54e9f5ba9e22e384b84"}, + {file = "pydantic_core-2.41.4-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1affa4798520b148d7182da0615d648e752de4ab1a9566b7471bc803d88a062d"}, + {file = "pydantic_core-2.41.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:7b74e18052fea4aa8dea2fb7dbc23d15439695da6cbe6cfc1b694af1115df09d"}, + {file = "pydantic_core-2.41.4-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:285b643d75c0e30abda9dc1077395624f314a37e3c09ca402d4015ef5979f1a2"}, + {file = "pydantic_core-2.41.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:f52679ff4218d713b3b33f88c89ccbf3a5c2c12ba665fb80ccc4192b4608dbab"}, + {file = "pydantic_core-2.41.4-cp313-cp313-win32.whl", hash = "sha256:ecde6dedd6fff127c273c76821bb754d793be1024bc33314a120f83a3c69460c"}, + {file = "pydantic_core-2.41.4-cp313-cp313-win_amd64.whl", hash = "sha256:d081a1f3800f05409ed868ebb2d74ac39dd0c1ff6c035b5162356d76030736d4"}, + {file = "pydantic_core-2.41.4-cp313-cp313-win_arm64.whl", hash = "sha256:f8e49c9c364a7edcbe2a310f12733aad95b022495ef2a8d653f645e5d20c1564"}, + {file = "pydantic_core-2.41.4-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:ed97fd56a561f5eb5706cebe94f1ad7c13b84d98312a05546f2ad036bafe87f4"}, + {file = "pydantic_core-2.41.4-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a870c307bf1ee91fc58a9a61338ff780d01bfae45922624816878dce784095d2"}, + {file = "pydantic_core-2.41.4-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d25e97bc1f5f8f7985bdc2335ef9e73843bb561eb1fa6831fdfc295c1c2061cf"}, + {file = "pydantic_core-2.41.4-cp313-cp313t-win_amd64.whl", hash = "sha256:d405d14bea042f166512add3091c1af40437c2e7f86988f3915fabd27b1e9cd2"}, + {file = "pydantic_core-2.41.4-cp313-cp313t-win_arm64.whl", hash = "sha256:19f3684868309db5263a11bace3c45d93f6f24afa2ffe75a647583df22a2ff89"}, + {file = "pydantic_core-2.41.4-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:e9205d97ed08a82ebb9a307e92914bb30e18cdf6f6b12ca4bedadb1588a0bfe1"}, + {file = "pydantic_core-2.41.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:82df1f432b37d832709fbcc0e24394bba04a01b6ecf1ee87578145c19cde12ac"}, + {file = "pydantic_core-2.41.4-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fc3b4cc4539e055cfa39a3763c939f9d409eb40e85813257dcd761985a108554"}, + {file = "pydantic_core-2.41.4-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b1eb1754fce47c63d2ff57fdb88c351a6c0150995890088b33767a10218eaa4e"}, + {file = "pydantic_core-2.41.4-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e6ab5ab30ef325b443f379ddb575a34969c333004fca5a1daa0133a6ffaad616"}, + {file = "pydantic_core-2.41.4-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:31a41030b1d9ca497634092b46481b937ff9397a86f9f51bd41c4767b6fc04af"}, + {file = "pydantic_core-2.41.4-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a44ac1738591472c3d020f61c6df1e4015180d6262ebd39bf2aeb52571b60f12"}, + {file = "pydantic_core-2.41.4-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d72f2b5e6e82ab8f94ea7d0d42f83c487dc159c5240d8f83beae684472864e2d"}, + {file = "pydantic_core-2.41.4-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:c4d1e854aaf044487d31143f541f7aafe7b482ae72a022c664b2de2e466ed0ad"}, + {file = "pydantic_core-2.41.4-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:b568af94267729d76e6ee5ececda4e283d07bbb28e8148bb17adad93d025d25a"}, + {file = "pydantic_core-2.41.4-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:6d55fb8b1e8929b341cc313a81a26e0d48aa3b519c1dbaadec3a6a2b4fcad025"}, + {file = "pydantic_core-2.41.4-cp314-cp314-win32.whl", hash = "sha256:5b66584e549e2e32a1398df11da2e0a7eff45d5c2d9db9d5667c5e6ac764d77e"}, + {file = "pydantic_core-2.41.4-cp314-cp314-win_amd64.whl", hash = "sha256:557a0aab88664cc552285316809cab897716a372afaf8efdbef756f8b890e894"}, + {file = "pydantic_core-2.41.4-cp314-cp314-win_arm64.whl", hash = "sha256:3f1ea6f48a045745d0d9f325989d8abd3f1eaf47dd00485912d1a3a63c623a8d"}, + {file = "pydantic_core-2.41.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6c1fe4c5404c448b13188dd8bd2ebc2bdd7e6727fa61ff481bcc2cca894018da"}, + {file = "pydantic_core-2.41.4-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:523e7da4d43b113bf8e7b49fa4ec0c35bf4fe66b2230bfc5c13cc498f12c6c3e"}, + {file = "pydantic_core-2.41.4-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5729225de81fb65b70fdb1907fcf08c75d498f4a6f15af005aabb1fdadc19dfa"}, + {file = "pydantic_core-2.41.4-cp314-cp314t-win_amd64.whl", hash = "sha256:de2cfbb09e88f0f795fd90cf955858fc2c691df65b1f21f0aa00b99f3fbc661d"}, + {file = "pydantic_core-2.41.4-cp314-cp314t-win_arm64.whl", hash = "sha256:d34f950ae05a83e0ede899c595f312ca976023ea1db100cd5aa188f7005e3ab0"}, + {file = "pydantic_core-2.41.4-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:646e76293345954acea6966149683047b7b2ace793011922208c8e9da12b0062"}, + {file = "pydantic_core-2.41.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:cc8e85a63085a137d286e2791037f5fdfff0aabb8b899483ca9c496dd5797338"}, + {file = "pydantic_core-2.41.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:692c622c8f859a17c156492783902d8370ac7e121a611bd6fe92cc71acf9ee8d"}, + {file = "pydantic_core-2.41.4-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d1e2906efb1031a532600679b424ef1d95d9f9fb507f813951f23320903adbd7"}, + {file = "pydantic_core-2.41.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e04e2f7f8916ad3ddd417a7abdd295276a0bf216993d9318a5d61cc058209166"}, + {file = "pydantic_core-2.41.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:df649916b81822543d1c8e0e1d079235f68acdc7d270c911e8425045a8cfc57e"}, + {file = "pydantic_core-2.41.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:66c529f862fdba70558061bb936fe00ddbaaa0c647fd26e4a4356ef1d6561891"}, + {file = "pydantic_core-2.41.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fc3b4c5a1fd3a311563ed866c2c9b62da06cb6398bee186484ce95c820db71cb"}, + {file = "pydantic_core-2.41.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:6e0fc40d84448f941df9b3334c4b78fe42f36e3bf631ad54c3047a0cdddc2514"}, + {file = "pydantic_core-2.41.4-cp39-cp39-musllinux_1_1_armv7l.whl", hash = "sha256:44e7625332683b6c1c8b980461475cde9595eff94447500e80716db89b0da005"}, + {file = "pydantic_core-2.41.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:170ee6835f6c71081d031ef1c3b4dc4a12b9efa6a9540f93f95b82f3c7571ae8"}, + {file = "pydantic_core-2.41.4-cp39-cp39-win32.whl", hash = "sha256:3adf61415efa6ce977041ba9745183c0e1f637ca849773afa93833e04b163feb"}, + {file = "pydantic_core-2.41.4-cp39-cp39-win_amd64.whl", hash = "sha256:a238dd3feee263eeaeb7dc44aea4ba1364682c4f9f9467e6af5596ba322c2332"}, + {file = "pydantic_core-2.41.4-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:a1b2cfec3879afb742a7b0bcfa53e4f22ba96571c9e54d6a3afe1052d17d843b"}, + {file = "pydantic_core-2.41.4-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:d175600d975b7c244af6eb9c9041f10059f20b8bbffec9e33fdd5ee3f67cdc42"}, + {file = "pydantic_core-2.41.4-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0f184d657fa4947ae5ec9c47bd7e917730fa1cbb78195037e32dcbab50aca5ee"}, + {file = "pydantic_core-2.41.4-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ed810568aeffed3edc78910af32af911c835cc39ebbfacd1f0ab5dd53028e5c"}, + {file = "pydantic_core-2.41.4-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:4f5d640aeebb438517150fdeec097739614421900e4a08db4a3ef38898798537"}, + {file = "pydantic_core-2.41.4-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:4a9ab037b71927babc6d9e7fc01aea9e66dc2a4a34dff06ef0724a4049629f94"}, + {file = "pydantic_core-2.41.4-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e4dab9484ec605c3016df9ad4fd4f9a390bc5d816a3b10c6550f8424bb80b18c"}, + {file = "pydantic_core-2.41.4-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bd8a5028425820731d8c6c098ab642d7b8b999758e24acae03ed38a66eca8335"}, + {file = "pydantic_core-2.41.4-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:1e5ab4fc177dd41536b3c32b2ea11380dd3d4619a385860621478ac2d25ceb00"}, + {file = "pydantic_core-2.41.4-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:3d88d0054d3fa11ce936184896bed3c1c5441d6fa483b498fac6a5d0dd6f64a9"}, + {file = "pydantic_core-2.41.4-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b2a054a8725f05b4b6503357e0ac1c4e8234ad3b0c2ac130d6ffc66f0e170e2"}, + {file = "pydantic_core-2.41.4-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b0d9db5a161c99375a0c68c058e227bee1d89303300802601d76a3d01f74e258"}, + {file = "pydantic_core-2.41.4-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:6273ea2c8ffdac7b7fda2653c49682db815aebf4a89243a6feccf5e36c18c347"}, + {file = "pydantic_core-2.41.4-pp310-pypy310_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:4c973add636efc61de22530b2ef83a65f39b6d6f656df97f678720e20de26caa"}, + {file = "pydantic_core-2.41.4-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:b69d1973354758007f46cf2d44a4f3d0933f10b6dc9bf15cf1356e037f6f731a"}, + {file = "pydantic_core-2.41.4-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:3619320641fd212aaf5997b6ca505e97540b7e16418f4a241f44cdf108ffb50d"}, + {file = "pydantic_core-2.41.4-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:491535d45cd7ad7e4a2af4a5169b0d07bebf1adfd164b0368da8aa41e19907a5"}, + {file = "pydantic_core-2.41.4-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:54d86c0cada6aba4ec4c047d0e348cbad7063b87ae0f005d9f8c9ad04d4a92a2"}, + {file = "pydantic_core-2.41.4-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eca1124aced216b2500dc2609eade086d718e8249cb9696660ab447d50a758bd"}, + {file = "pydantic_core-2.41.4-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6c9024169becccf0cb470ada03ee578d7348c119a0d42af3dcf9eda96e3a247c"}, + {file = "pydantic_core-2.41.4-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:26895a4268ae5a2849269f4991cdc97236e4b9c010e51137becf25182daac405"}, + {file = "pydantic_core-2.41.4-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:ca4df25762cf71308c446e33c9b1fdca2923a3f13de616e2a949f38bf21ff5a8"}, + {file = "pydantic_core-2.41.4-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:5a28fcedd762349519276c36634e71853b4541079cab4acaaac60c4421827308"}, + {file = "pydantic_core-2.41.4-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:c173ddcd86afd2535e2b695217e82191580663a1d1928239f877f5a1649ef39f"}, + {file = "pydantic_core-2.41.4.tar.gz", hash = "sha256:70e47929a9d4a1905a67e4b687d5946026390568a8e952b92824118063cee4d5"}, +] + +[package.dependencies] +typing-extensions = ">=4.14.1" + +[[package]] +name = "pydantic-settings" +version = "2.11.0" +description = "Settings management using Pydantic" +optional = false +python-versions = ">=3.9" +groups = ["main"] +markers = "python_version >= \"3.10\"" +files = [ + {file = "pydantic_settings-2.11.0-py3-none-any.whl", hash = "sha256:fe2cea3413b9530d10f3a5875adffb17ada5c1e1bab0b2885546d7310415207c"}, + {file = "pydantic_settings-2.11.0.tar.gz", hash = "sha256:d0e87a1c7d33593beb7194adb8470fc426e95ba02af83a0f23474a04c9a08180"}, +] + +[package.dependencies] +pydantic = ">=2.7.0" +python-dotenv = ">=0.21.0" +typing-inspection = ">=0.4.0" + +[package.extras] +aws-secrets-manager = ["boto3 (>=1.35.0)", "boto3-stubs[secretsmanager]"] +azure-key-vault = ["azure-identity (>=1.16.0)", "azure-keyvault-secrets (>=4.8.0)"] +gcp-secret-manager = ["google-cloud-secret-manager (>=2.23.1)"] +toml = ["tomli (>=2.0.1)"] +yaml = ["pyyaml (>=6.0.1)"] + +[[package]] +name = "pyfakefs" +version = "5.10.2" +description = "Implements a fake file system that mocks the Python file system modules." +optional = false +python-versions = ">=3.7" +groups = ["test"] +files = [ + {file = "pyfakefs-5.10.2-py3-none-any.whl", hash = "sha256:6ff0e84653a71efc6a73f9ee839c3141e3a7cdf4e1fb97666f82ac5b24308d64"}, + {file = "pyfakefs-5.10.2.tar.gz", hash = "sha256:8ae0e5421e08de4e433853a4609a06a1835f4bc2a3ce13b54f36713a897474ba"}, +] + +[[package]] +name = "pygments" +version = "2.19.2" +description = "Pygments is a syntax highlighting package written in Python." +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b"}, + {file = "pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887"}, +] + +[package.extras] +windows-terminal = ["colorama (>=0.4.6)"] [[package]] name = "pyinstaller" -version = "5.11.0" +version = "6.19.0" description = "PyInstaller bundles a Python application and all its dependencies into a single package." -category = "dev" optional = false -python-versions = "<3.12,>=3.7" +python-versions = "<3.15,>=3.8" +groups = ["executable"] +markers = "python_version < \"3.15\"" files = [ - {file = "pyinstaller-5.11.0-py3-none-macosx_10_13_universal2.whl", hash = "sha256:8454bac8f3cb2219a3ce2227fd039bdaf943dcba60e8c55732958ea3a6d81263"}, - {file = "pyinstaller-5.11.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:b3c6299fd7526c6ca87ea5f9017fb1928d47046df0b9f983d6bbd893801010dc"}, - {file = "pyinstaller-5.11.0-py3-none-manylinux2014_i686.whl", hash = "sha256:e359571327bbef434fc61324891399f9117efbb685b5065234eebb01713650a8"}, - {file = "pyinstaller-5.11.0-py3-none-manylinux2014_ppc64le.whl", hash = "sha256:a445a91b85c9a1ea3985268643a674900dd86f244cc4be4ff4ec4c6367ff99a9"}, - {file = "pyinstaller-5.11.0-py3-none-manylinux2014_s390x.whl", hash = "sha256:2a1fe6d0da22f207cfa4b3221fe365503cba071c77aac19f76a75503f67d9ff9"}, - {file = "pyinstaller-5.11.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:b4cac0e7b0d63c6a869843113008f59fd5b38b2959ffa6059e7fac4bb05de92b"}, - {file = "pyinstaller-5.11.0-py3-none-musllinux_1_1_aarch64.whl", hash = "sha256:0af9d11a09ce217d32f95c79c984054457b310671387ff32bae1496876308556"}, - {file = "pyinstaller-5.11.0-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:b8a4f6834e5c85150948e22c74dd3ab8b98aa4ccdf964d880ac14d2f78d9c1a4"}, - {file = "pyinstaller-5.11.0-py3-none-win32.whl", hash = "sha256:049cdc3524aefb5ca015a63d2c81b6bf1567cc818ac066859fbfde702c6165d3"}, - {file = "pyinstaller-5.11.0-py3-none-win_amd64.whl", hash = "sha256:42fdea67e4c2217cedd54d17d1d402736df3ba718db2b497df65df5a68ae4f93"}, - {file = "pyinstaller-5.11.0-py3-none-win_arm64.whl", hash = "sha256:036a062a228af41f6bb6370a4e87cef34858cc839200a07ace7f8738ef64ad86"}, - {file = "pyinstaller-5.11.0.tar.gz", hash = "sha256:cb87cee0b3c81ccd74d4bf3f4faf03b5e1e39bb91f1a894b2ce4cd22363bf779"}, + {file = "pyinstaller-6.19.0-py3-none-macosx_10_13_universal2.whl", hash = "sha256:4190e76b74f0c4b5c5f11ac360928cd2e36ec8e3194d437bf6b8648c7bc0c134"}, + {file = "pyinstaller-6.19.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:8bd68abd812d8a6ba33b9f1810e91fee0f325969733721b78151f0065319ca11"}, + {file = "pyinstaller-6.19.0-py3-none-manylinux2014_i686.whl", hash = "sha256:1ec54ef967996ca61dacba676227e2b23219878ccce5ee9d6f3aada7b8ed8abf"}, + {file = "pyinstaller-6.19.0-py3-none-manylinux2014_ppc64le.whl", hash = "sha256:4ab2bb52e58448e14ddf9450601bdedd66800465043501c1d8f1cab87b60b122"}, + {file = "pyinstaller-6.19.0-py3-none-manylinux2014_s390x.whl", hash = "sha256:da6d5c6391ccefe73554b9fa29b86001c8e378e0f20c2a4004f836ba537eff63"}, + {file = "pyinstaller-6.19.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:a0fc5f6b3c55aa54353f0c74ffa59b1115433c1850c6f655d62b461a2ed6cbbe"}, + {file = "pyinstaller-6.19.0-py3-none-musllinux_1_1_aarch64.whl", hash = "sha256:e649ba6bd1b0b89b210ad92adb5fbdc8a42dd2c5ca4f72ef3a0bfec83a424b83"}, + {file = "pyinstaller-6.19.0-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:481a909c8e60c8692fc60fcb1344d984b44b943f8bc9682f2fcdae305ad297e6"}, + {file = "pyinstaller-6.19.0-py3-none-win32.whl", hash = "sha256:3c5c251054fe4cfaa04c34a363dcfbf811545438cb7198304cd444756bc2edd2"}, + {file = "pyinstaller-6.19.0-py3-none-win_amd64.whl", hash = "sha256:b5bb6536c6560330d364d91522250f254b107cf69129d9cbcd0e6727c570be33"}, + {file = "pyinstaller-6.19.0-py3-none-win_arm64.whl", hash = "sha256:c2d5a539b0bfe6159d5522c8c70e1c0e487f22c2badae0f97d45246223b798ea"}, + {file = "pyinstaller-6.19.0.tar.gz", hash = "sha256:ec73aeb8bd9b7f2f1240d328a4542e90b3c6e6fbc106014778431c616592a865"}, ] [package.dependencies] altgraph = "*" -importlib-metadata = {version = ">=1.4", markers = "python_version < \"3.8\""} +importlib-metadata = {version = ">=4.6", markers = "python_version < \"3.10\""} macholib = {version = ">=1.8", markers = "sys_platform == \"darwin\""} +packaging = ">=22.0" pefile = {version = ">=2022.5.30", markers = "sys_platform == \"win32\""} -pyinstaller-hooks-contrib = ">=2021.4" -pywin32-ctypes = {version = ">=0.2.0", markers = "sys_platform == \"win32\""} +pyinstaller-hooks-contrib = ">=2026.0" +pywin32-ctypes = {version = ">=0.2.1", markers = "sys_platform == \"win32\""} setuptools = ">=42.0.0" [package.extras] -encryption = ["tinyaes (>=1.0.0)"] +completion = ["argcomplete"] hook-testing = ["execnet (>=1.5.0)", "psutil", "pytest (>=2.7.3)"] [[package]] name = "pyinstaller-hooks-contrib" -version = "2023.3" +version = "2026.0" description = "Community maintained hooks for PyInstaller" -category = "dev" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" +groups = ["executable"] +markers = "python_version < \"3.15\"" +files = [ + {file = "pyinstaller_hooks_contrib-2026.0-py3-none-any.whl", hash = "sha256:0590db8edeba3e6c30c8474937021f5cd39c0602b4d10f74a064c73911efaca5"}, + {file = "pyinstaller_hooks_contrib-2026.0.tar.gz", hash = "sha256:0120893de491a000845470ca9c0b39284731ac6bace26f6849dea9627aaed48e"}, +] + +[package.dependencies] +importlib_metadata = {version = ">=4.6", markers = "python_version < \"3.10\""} +packaging = ">=22.0" +setuptools = ">=42.0.0" + +[[package]] +name = "pyjwt" +version = "2.10.1" +description = "JSON Web Token implementation in Python" +optional = false +python-versions = ">=3.9" +groups = ["main"] files = [ - {file = "pyinstaller-hooks-contrib-2023.3.tar.gz", hash = "sha256:bb39e1038e3e0972420455e0b39cd9dce73f3d80acaf4bf2b3615fea766ff370"}, - {file = "pyinstaller_hooks_contrib-2023.3-py2.py3-none-any.whl", hash = "sha256:062ad7a1746e1cfc24d3a8c4be4e606fced3b123bda7d419f14fcf7507804b07"}, + {file = "PyJWT-2.10.1-py3-none-any.whl", hash = "sha256:dcdd193e30abefd5debf142f9adfcdd2b58004e644f25406ffaebd50bd98dacb"}, + {file = "pyjwt-2.10.1.tar.gz", hash = "sha256:3cc5772eb20009233caf06e9d8a0577824723b44e6648ee0a2aedb6cf9381953"}, ] +[package.dependencies] +cryptography = {version = ">=3.4.0", optional = true, markers = "extra == \"crypto\""} + +[package.extras] +crypto = ["cryptography (>=3.4.0)"] +dev = ["coverage[toml] (==5.0.4)", "cryptography (>=3.4.0)", "pre-commit", "pytest (>=6.0.0,<7.0.0)", "sphinx", "sphinx-rtd-theme", "zope.interface"] +docs = ["sphinx", "sphinx-rtd-theme", "zope.interface"] +tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] + [[package]] name = "pytest" -version = "7.3.1" +version = "7.3.2" description = "pytest: simple powerful testing with Python" -category = "dev" optional = false python-versions = ">=3.7" +groups = ["test"] files = [ - {file = "pytest-7.3.1-py3-none-any.whl", hash = "sha256:3799fa815351fea3a5e96ac7e503a96fa51cc9942c3753cda7651b93c1cfa362"}, - {file = "pytest-7.3.1.tar.gz", hash = "sha256:434afafd78b1d78ed0addf160ad2b77a30d35d4bdf8af234fe621919d9ed15e3"}, + {file = "pytest-7.3.2-py3-none-any.whl", hash = "sha256:cdcbd012c9312258922f8cd3f1b62a6580fdced17db6014896053d47cddf9295"}, + {file = "pytest-7.3.2.tar.gz", hash = "sha256:ee990a3cc55ba808b80795a79944756f315c67c12b56abd3ac993a7b8c17030b"}, ] [package.dependencies] colorama = {version = "*", markers = "sys_platform == \"win32\""} exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} -importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} iniconfig = "*" packaging = "*" pluggy = ">=0.12,<2.0" tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} [package.extras] -testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "xmlschema"] +testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] [[package]] name = "pytest-mock" version = "3.10.0" description = "Thin-wrapper around the mock package for easier use with pytest" -category = "dev" optional = false python-versions = ">=3.7" +groups = ["test"] files = [ {file = "pytest-mock-3.10.0.tar.gz", hash = "sha256:fbbdb085ef7c252a326fd8cdcac0aa3b1333d8811f131bdcc701002e1be7ed4f"}, {file = "pytest_mock-3.10.0-py3-none-any.whl", hash = "sha256:f4c973eeae0282963eb293eb173ce91b091a79c1334455acfac9ddee8a1c784b"}, @@ -583,96 +1251,208 @@ dev = ["pre-commit", "pytest-asyncio", "tox"] [[package]] name = "python-dateutil" -version = "2.8.2" +version = "2.9.0.post0" description = "Extensions to the standard Python datetime module" -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +groups = ["main"] files = [ - {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, - {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, + {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, + {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, ] [package.dependencies] six = ">=1.5" [[package]] -name = "pywin32-ctypes" -version = "0.2.0" -description = "" -category = "dev" +name = "python-dotenv" +version = "1.1.1" +description = "Read key-value pairs from a .env file and set them as environment variables" +optional = false +python-versions = ">=3.9" +groups = ["main"] +markers = "python_version >= \"3.10\"" +files = [ + {file = "python_dotenv-1.1.1-py3-none-any.whl", hash = "sha256:31f23644fe2602f88ff55e1f5c79ba497e01224ee7737937930c448e4d0e24dc"}, + {file = "python_dotenv-1.1.1.tar.gz", hash = "sha256:a8a6399716257f45be6a007360200409fce5cda2661e3dec71d23dc15f6189ab"}, +] + +[package.extras] +cli = ["click (>=5.0)"] + +[[package]] +name = "python-multipart" +version = "0.0.22" +description = "A streaming multipart parser for Python" +optional = false +python-versions = ">=3.10" +groups = ["main"] +markers = "python_version >= \"3.10\"" +files = [ + {file = "python_multipart-0.0.22-py3-none-any.whl", hash = "sha256:2b2cd894c83d21bf49d702499531c7bafd057d730c201782048f7945d82de155"}, + {file = "python_multipart-0.0.22.tar.gz", hash = "sha256:7340bef99a7e0032613f56dc36027b959fd3b30a787ed62d310e951f7c3a3a58"}, +] + +[[package]] +name = "pywin32" +version = "311" +description = "Python for Window Extensions" optional = false python-versions = "*" +groups = ["main"] +markers = "python_version >= \"3.10\" and sys_platform == \"win32\"" files = [ - {file = "pywin32-ctypes-0.2.0.tar.gz", hash = "sha256:24ffc3b341d457d48e8922352130cf2644024a4ff09762a2261fd34c36ee5942"}, - {file = "pywin32_ctypes-0.2.0-py2.py3-none-any.whl", hash = "sha256:9dc2d991b3479cc2df15930958b674a48a227d5361d413827a4cfd0b5876fc98"}, + {file = "pywin32-311-cp310-cp310-win32.whl", hash = "sha256:d03ff496d2a0cd4a5893504789d4a15399133fe82517455e78bad62efbb7f0a3"}, + {file = "pywin32-311-cp310-cp310-win_amd64.whl", hash = "sha256:797c2772017851984b97180b0bebe4b620bb86328e8a884bb626156295a63b3b"}, + {file = "pywin32-311-cp310-cp310-win_arm64.whl", hash = "sha256:0502d1facf1fed4839a9a51ccbcc63d952cf318f78ffc00a7e78528ac27d7a2b"}, + {file = "pywin32-311-cp311-cp311-win32.whl", hash = "sha256:184eb5e436dea364dcd3d2316d577d625c0351bf237c4e9a5fabbcfa5a58b151"}, + {file = "pywin32-311-cp311-cp311-win_amd64.whl", hash = "sha256:3ce80b34b22b17ccbd937a6e78e7225d80c52f5ab9940fe0506a1a16f3dab503"}, + {file = "pywin32-311-cp311-cp311-win_arm64.whl", hash = "sha256:a733f1388e1a842abb67ffa8e7aad0e70ac519e09b0f6a784e65a136ec7cefd2"}, + {file = "pywin32-311-cp312-cp312-win32.whl", hash = "sha256:750ec6e621af2b948540032557b10a2d43b0cee2ae9758c54154d711cc852d31"}, + {file = "pywin32-311-cp312-cp312-win_amd64.whl", hash = "sha256:b8c095edad5c211ff31c05223658e71bf7116daa0ecf3ad85f3201ea3190d067"}, + {file = "pywin32-311-cp312-cp312-win_arm64.whl", hash = "sha256:e286f46a9a39c4a18b319c28f59b61de793654af2f395c102b4f819e584b5852"}, + {file = "pywin32-311-cp313-cp313-win32.whl", hash = "sha256:f95ba5a847cba10dd8c4d8fefa9f2a6cf283b8b88ed6178fa8a6c1ab16054d0d"}, + {file = "pywin32-311-cp313-cp313-win_amd64.whl", hash = "sha256:718a38f7e5b058e76aee1c56ddd06908116d35147e133427e59a3983f703a20d"}, + {file = "pywin32-311-cp313-cp313-win_arm64.whl", hash = "sha256:7b4075d959648406202d92a2310cb990fea19b535c7f4a78d3f5e10b926eeb8a"}, + {file = "pywin32-311-cp314-cp314-win32.whl", hash = "sha256:b7a2c10b93f8986666d0c803ee19b5990885872a7de910fc460f9b0c2fbf92ee"}, + {file = "pywin32-311-cp314-cp314-win_amd64.whl", hash = "sha256:3aca44c046bd2ed8c90de9cb8427f581c479e594e99b5c0bb19b29c10fd6cb87"}, + {file = "pywin32-311-cp314-cp314-win_arm64.whl", hash = "sha256:a508e2d9025764a8270f93111a970e1d0fbfc33f4153b388bb649b7eec4f9b42"}, + {file = "pywin32-311-cp38-cp38-win32.whl", hash = "sha256:6c6f2969607b5023b0d9ce2541f8d2cbb01c4f46bc87456017cf63b73f1e2d8c"}, + {file = "pywin32-311-cp38-cp38-win_amd64.whl", hash = "sha256:c8015b09fb9a5e188f83b7b04de91ddca4658cee2ae6f3bc483f0b21a77ef6cd"}, + {file = "pywin32-311-cp39-cp39-win32.whl", hash = "sha256:aba8f82d551a942cb20d4a83413ccbac30790b50efb89a75e4f586ac0bb8056b"}, + {file = "pywin32-311-cp39-cp39-win_amd64.whl", hash = "sha256:e0c4cfb0621281fe40387df582097fd796e80430597cb9944f0ae70447bacd91"}, + {file = "pywin32-311-cp39-cp39-win_arm64.whl", hash = "sha256:62ea666235135fee79bb154e695f3ff67370afefd71bd7fea7512fc70ef31e3d"}, +] + +[[package]] +name = "pywin32-ctypes" +version = "0.2.3" +description = "A (partial) reimplementation of pywin32 using ctypes/cffi" +optional = false +python-versions = ">=3.6" +groups = ["executable"] +markers = "python_version < \"3.15\" and sys_platform == \"win32\"" +files = [ + {file = "pywin32-ctypes-0.2.3.tar.gz", hash = "sha256:d162dc04946d704503b2edc4d55f3dba5c1d539ead017afa00142c38b9885755"}, + {file = "pywin32_ctypes-0.2.3-py3-none-any.whl", hash = "sha256:8a1513379d709975552d202d942d9837758905c8d01eb82b8bcc30918929e7b8"}, ] [[package]] name = "pyyaml" -version = "6.0" +version = "6.0.3" description = "YAML parser and emitter for Python" -category = "main" optional = false -python-versions = ">=3.6" +python-versions = ">=3.8" +groups = ["main", "test"] files = [ - {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"}, - {file = "PyYAML-6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c"}, - {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc"}, - {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b"}, - {file = "PyYAML-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"}, - {file = "PyYAML-6.0-cp310-cp310-win32.whl", hash = "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513"}, - {file = "PyYAML-6.0-cp310-cp310-win_amd64.whl", hash = "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a"}, - {file = "PyYAML-6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358"}, - {file = "PyYAML-6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782"}, - {file = "PyYAML-6.0-cp311-cp311-win32.whl", hash = "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7"}, - {file = "PyYAML-6.0-cp311-cp311-win_amd64.whl", hash = "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf"}, - {file = "PyYAML-6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86"}, - {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f"}, - {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92"}, - {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4"}, - {file = "PyYAML-6.0-cp36-cp36m-win32.whl", hash = "sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293"}, - {file = "PyYAML-6.0-cp36-cp36m-win_amd64.whl", hash = "sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57"}, - {file = "PyYAML-6.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c"}, - {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0"}, - {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4"}, - {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9"}, - {file = "PyYAML-6.0-cp37-cp37m-win32.whl", hash = "sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737"}, - {file = "PyYAML-6.0-cp37-cp37m-win_amd64.whl", hash = "sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d"}, - {file = "PyYAML-6.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b"}, - {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba"}, - {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34"}, - {file = "PyYAML-6.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287"}, - {file = "PyYAML-6.0-cp38-cp38-win32.whl", hash = "sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78"}, - {file = "PyYAML-6.0-cp38-cp38-win_amd64.whl", hash = "sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07"}, - {file = "PyYAML-6.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b"}, - {file = "PyYAML-6.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174"}, - {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803"}, - {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3"}, - {file = "PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0"}, - {file = "PyYAML-6.0-cp39-cp39-win32.whl", hash = "sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb"}, - {file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"}, - {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, + {file = "PyYAML-6.0.3-cp38-cp38-macosx_10_13_x86_64.whl", hash = "sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f"}, + {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4"}, + {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3"}, + {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6"}, + {file = "PyYAML-6.0.3-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369"}, + {file = "PyYAML-6.0.3-cp38-cp38-win32.whl", hash = "sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295"}, + {file = "PyYAML-6.0.3-cp38-cp38-win_amd64.whl", hash = "sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b"}, + {file = "pyyaml-6.0.3-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b"}, + {file = "pyyaml-6.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956"}, + {file = "pyyaml-6.0.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8"}, + {file = "pyyaml-6.0.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198"}, + {file = "pyyaml-6.0.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b"}, + {file = "pyyaml-6.0.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0"}, + {file = "pyyaml-6.0.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69"}, + {file = "pyyaml-6.0.3-cp310-cp310-win32.whl", hash = "sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e"}, + {file = "pyyaml-6.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c"}, + {file = "pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e"}, + {file = "pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824"}, + {file = "pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c"}, + {file = "pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00"}, + {file = "pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d"}, + {file = "pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a"}, + {file = "pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4"}, + {file = "pyyaml-6.0.3-cp311-cp311-win32.whl", hash = "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b"}, + {file = "pyyaml-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf"}, + {file = "pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196"}, + {file = "pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0"}, + {file = "pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28"}, + {file = "pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c"}, + {file = "pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc"}, + {file = "pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e"}, + {file = "pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea"}, + {file = "pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5"}, + {file = "pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b"}, + {file = "pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd"}, + {file = "pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8"}, + {file = "pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1"}, + {file = "pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c"}, + {file = "pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5"}, + {file = "pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6"}, + {file = "pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6"}, + {file = "pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be"}, + {file = "pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26"}, + {file = "pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c"}, + {file = "pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb"}, + {file = "pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac"}, + {file = "pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310"}, + {file = "pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7"}, + {file = "pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788"}, + {file = "pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5"}, + {file = "pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764"}, + {file = "pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35"}, + {file = "pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac"}, + {file = "pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3"}, + {file = "pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3"}, + {file = "pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba"}, + {file = "pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c"}, + {file = "pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702"}, + {file = "pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c"}, + {file = "pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065"}, + {file = "pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65"}, + {file = "pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9"}, + {file = "pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b"}, + {file = "pyyaml-6.0.3-cp39-cp39-macosx_10_13_x86_64.whl", hash = "sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da"}, + {file = "pyyaml-6.0.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917"}, + {file = "pyyaml-6.0.3-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9"}, + {file = "pyyaml-6.0.3-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5"}, + {file = "pyyaml-6.0.3-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a"}, + {file = "pyyaml-6.0.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926"}, + {file = "pyyaml-6.0.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7"}, + {file = "pyyaml-6.0.3-cp39-cp39-win32.whl", hash = "sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0"}, + {file = "pyyaml-6.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007"}, + {file = "pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f"}, ] +[[package]] +name = "referencing" +version = "0.37.0" +description = "JSON Referencing + Python" +optional = false +python-versions = ">=3.10" +groups = ["main"] +markers = "python_version >= \"3.10\"" +files = [ + {file = "referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231"}, + {file = "referencing-0.37.0.tar.gz", hash = "sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8"}, +] + +[package.dependencies] +attrs = ">=22.2.0" +rpds-py = ">=0.7.0" +typing-extensions = {version = ">=4.4.0", markers = "python_version < \"3.13\""} + [[package]] name = "requests" -version = "2.31.0" +version = "2.32.5" description = "Python HTTP for Humans." -category = "main" optional = false -python-versions = ">=3.7" +python-versions = ">=3.9" +groups = ["main", "test"] files = [ - {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, - {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, + {file = "requests-2.32.5-py3-none-any.whl", hash = "sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6"}, + {file = "requests-2.32.5.tar.gz", hash = "sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf"}, ] [package.dependencies] certifi = ">=2017.4.17" -charset-normalizer = ">=2,<4" +charset_normalizer = ">=2,<4" idna = ">=2.5,<4" urllib3 = ">=1.21.1,<3" @@ -682,177 +1462,539 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] [[package]] name = "responses" -version = "0.23.1" +version = "0.26.0" description = "A utility library for mocking out the `requests` Python library." -category = "dev" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" +groups = ["test"] files = [ - {file = "responses-0.23.1-py3-none-any.whl", hash = "sha256:8a3a5915713483bf353b6f4079ba8b2a29029d1d1090a503c70b0dc5d9d0c7bd"}, - {file = "responses-0.23.1.tar.gz", hash = "sha256:c4d9aa9fc888188f0c673eff79a8dadbe2e75b7fe879dc80a221a06e0a68138f"}, + {file = "responses-0.26.0-py3-none-any.whl", hash = "sha256:03ec4409088cd5c66b71ecbbbd27fe2c58ddfad801c66203457b3e6a04868c37"}, + {file = "responses-0.26.0.tar.gz", hash = "sha256:c7f6923e6343ef3682816ba421c006626777893cb0d5e1434f674b649bac9eb4"}, ] [package.dependencies] pyyaml = "*" -requests = ">=2.22.0,<3.0" -types-PyYAML = "*" -typing-extensions = {version = "*", markers = "python_version < \"3.8\""} -urllib3 = ">=1.25.10" +requests = ">=2.30.0,<3.0" +urllib3 = ">=1.25.10,<3.0" + +[package.extras] +tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asyncio", "pytest-cov", "pytest-httpserver", "tomli ; python_version < \"3.11\"", "tomli-w", "types-PyYAML", "types-requests"] + +[[package]] +name = "rich" +version = "13.9.4" +description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" +optional = false +python-versions = ">=3.8.0" +groups = ["main"] +files = [ + {file = "rich-13.9.4-py3-none-any.whl", hash = "sha256:6049d5e6ec054bf2779ab3358186963bac2ea89175919d699e378b99738c2a90"}, + {file = "rich-13.9.4.tar.gz", hash = "sha256:439594978a49a09530cff7ebc4b5c7103ef57baf48d5ea3184f21d9a2befa098"}, +] + +[package.dependencies] +markdown-it-py = ">=2.2.0" +pygments = ">=2.13.0,<3.0.0" +typing-extensions = {version = ">=4.0.0,<5.0", markers = "python_version < \"3.11\""} [package.extras] -tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asyncio", "pytest-cov", "pytest-httpserver", "tomli", "tomli-w", "types-requests"] +jupyter = ["ipywidgets (>=7.5.1,<9)"] + +[[package]] +name = "rpds-py" +version = "0.27.1" +description = "Python bindings to Rust's persistent data structures (rpds)" +optional = false +python-versions = ">=3.9" +groups = ["main"] +markers = "python_version >= \"3.10\"" +files = [ + {file = "rpds_py-0.27.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:68afeec26d42ab3b47e541b272166a0b4400313946871cba3ed3a4fc0cab1cef"}, + {file = "rpds_py-0.27.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:74e5b2f7bb6fa38b1b10546d27acbacf2a022a8b5543efb06cfebc72a59c85be"}, + {file = "rpds_py-0.27.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9024de74731df54546fab0bfbcdb49fae19159ecaecfc8f37c18d2c7e2c0bd61"}, + {file = "rpds_py-0.27.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:31d3ebadefcd73b73928ed0b2fd696f7fefda8629229f81929ac9c1854d0cffb"}, + {file = "rpds_py-0.27.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b2e7f8f169d775dd9092a1743768d771f1d1300453ddfe6325ae3ab5332b4657"}, + {file = "rpds_py-0.27.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3d905d16f77eb6ab2e324e09bfa277b4c8e5e6b8a78a3e7ff8f3cdf773b4c013"}, + {file = "rpds_py-0.27.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:50c946f048209e6362e22576baea09193809f87687a95a8db24e5fbdb307b93a"}, + {file = "rpds_py-0.27.1-cp310-cp310-manylinux_2_31_riscv64.whl", hash = "sha256:3deab27804d65cd8289eb814c2c0e807c4b9d9916c9225e363cb0cf875eb67c1"}, + {file = "rpds_py-0.27.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8b61097f7488de4be8244c89915da8ed212832ccf1e7c7753a25a394bf9b1f10"}, + {file = "rpds_py-0.27.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:8a3f29aba6e2d7d90528d3c792555a93497fe6538aa65eb675b44505be747808"}, + {file = "rpds_py-0.27.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:dd6cd0485b7d347304067153a6dc1d73f7d4fd995a396ef32a24d24b8ac63ac8"}, + {file = "rpds_py-0.27.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:6f4461bf931108c9fa226ffb0e257c1b18dc2d44cd72b125bec50ee0ab1248a9"}, + {file = "rpds_py-0.27.1-cp310-cp310-win32.whl", hash = "sha256:ee5422d7fb21f6a00c1901bf6559c49fee13a5159d0288320737bbf6585bd3e4"}, + {file = "rpds_py-0.27.1-cp310-cp310-win_amd64.whl", hash = "sha256:3e039aabf6d5f83c745d5f9a0a381d031e9ed871967c0a5c38d201aca41f3ba1"}, + {file = "rpds_py-0.27.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:be898f271f851f68b318872ce6ebebbc62f303b654e43bf72683dbdc25b7c881"}, + {file = "rpds_py-0.27.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:62ac3d4e3e07b58ee0ddecd71d6ce3b1637de2d373501412df395a0ec5f9beb5"}, + {file = "rpds_py-0.27.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4708c5c0ceb2d034f9991623631d3d23cb16e65c83736ea020cdbe28d57c0a0e"}, + {file = "rpds_py-0.27.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:abfa1171a9952d2e0002aba2ad3780820b00cc3d9c98c6630f2e93271501f66c"}, + {file = "rpds_py-0.27.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4b507d19f817ebaca79574b16eb2ae412e5c0835542c93fe9983f1e432aca195"}, + {file = "rpds_py-0.27.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:168b025f8fd8d8d10957405f3fdcef3dc20f5982d398f90851f4abc58c566c52"}, + {file = "rpds_py-0.27.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cb56c6210ef77caa58e16e8c17d35c63fe3f5b60fd9ba9d424470c3400bcf9ed"}, + {file = "rpds_py-0.27.1-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:d252f2d8ca0195faa707f8eb9368955760880b2b42a8ee16d382bf5dd807f89a"}, + {file = "rpds_py-0.27.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6e5e54da1e74b91dbc7996b56640f79b195d5925c2b78efaa8c5d53e1d88edde"}, + {file = "rpds_py-0.27.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ffce0481cc6e95e5b3f0a47ee17ffbd234399e6d532f394c8dce320c3b089c21"}, + {file = "rpds_py-0.27.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:a205fdfe55c90c2cd8e540ca9ceba65cbe6629b443bc05db1f590a3db8189ff9"}, + {file = "rpds_py-0.27.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:689fb5200a749db0415b092972e8eba85847c23885c8543a8b0f5c009b1a5948"}, + {file = "rpds_py-0.27.1-cp311-cp311-win32.whl", hash = "sha256:3182af66048c00a075010bc7f4860f33913528a4b6fc09094a6e7598e462fe39"}, + {file = "rpds_py-0.27.1-cp311-cp311-win_amd64.whl", hash = "sha256:b4938466c6b257b2f5c4ff98acd8128ec36b5059e5c8f8372d79316b1c36bb15"}, + {file = "rpds_py-0.27.1-cp311-cp311-win_arm64.whl", hash = "sha256:2f57af9b4d0793e53266ee4325535a31ba48e2f875da81a9177c9926dfa60746"}, + {file = "rpds_py-0.27.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:ae2775c1973e3c30316892737b91f9283f9908e3cc7625b9331271eaaed7dc90"}, + {file = "rpds_py-0.27.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2643400120f55c8a96f7c9d858f7be0c88d383cd4653ae2cf0d0c88f668073e5"}, + {file = "rpds_py-0.27.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:16323f674c089b0360674a4abd28d5042947d54ba620f72514d69be4ff64845e"}, + {file = "rpds_py-0.27.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9a1f4814b65eacac94a00fc9a526e3fdafd78e439469644032032d0d63de4881"}, + {file = "rpds_py-0.27.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7ba32c16b064267b22f1850a34051121d423b6f7338a12b9459550eb2096e7ec"}, + {file = "rpds_py-0.27.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e5c20f33fd10485b80f65e800bbe5f6785af510b9f4056c5a3c612ebc83ba6cb"}, + {file = "rpds_py-0.27.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:466bfe65bd932da36ff279ddd92de56b042f2266d752719beb97b08526268ec5"}, + {file = "rpds_py-0.27.1-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:41e532bbdcb57c92ba3be62c42e9f096431b4cf478da9bc3bc6ce5c38ab7ba7a"}, + {file = "rpds_py-0.27.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f149826d742b406579466283769a8ea448eed82a789af0ed17b0cd5770433444"}, + {file = "rpds_py-0.27.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:80c60cfb5310677bd67cb1e85a1e8eb52e12529545441b43e6f14d90b878775a"}, + {file = "rpds_py-0.27.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:7ee6521b9baf06085f62ba9c7a3e5becffbc32480d2f1b351559c001c38ce4c1"}, + {file = "rpds_py-0.27.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a512c8263249a9d68cac08b05dd59d2b3f2061d99b322813cbcc14c3c7421998"}, + {file = "rpds_py-0.27.1-cp312-cp312-win32.whl", hash = "sha256:819064fa048ba01b6dadc5116f3ac48610435ac9a0058bbde98e569f9e785c39"}, + {file = "rpds_py-0.27.1-cp312-cp312-win_amd64.whl", hash = "sha256:d9199717881f13c32c4046a15f024971a3b78ad4ea029e8da6b86e5aa9cf4594"}, + {file = "rpds_py-0.27.1-cp312-cp312-win_arm64.whl", hash = "sha256:33aa65b97826a0e885ef6e278fbd934e98cdcfed80b63946025f01e2f5b29502"}, + {file = "rpds_py-0.27.1-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:e4b9fcfbc021633863a37e92571d6f91851fa656f0180246e84cbd8b3f6b329b"}, + {file = "rpds_py-0.27.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:1441811a96eadca93c517d08df75de45e5ffe68aa3089924f963c782c4b898cf"}, + {file = "rpds_py-0.27.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:55266dafa22e672f5a4f65019015f90336ed31c6383bd53f5e7826d21a0e0b83"}, + {file = "rpds_py-0.27.1-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d78827d7ac08627ea2c8e02c9e5b41180ea5ea1f747e9db0915e3adf36b62dcf"}, + {file = "rpds_py-0.27.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae92443798a40a92dc5f0b01d8a7c93adde0c4dc965310a29ae7c64d72b9fad2"}, + {file = "rpds_py-0.27.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c46c9dd2403b66a2a3b9720ec4b74d4ab49d4fabf9f03dfdce2d42af913fe8d0"}, + {file = "rpds_py-0.27.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2efe4eb1d01b7f5f1939f4ef30ecea6c6b3521eec451fb93191bf84b2a522418"}, + {file = "rpds_py-0.27.1-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:15d3b4d83582d10c601f481eca29c3f138d44c92187d197aff663a269197c02d"}, + {file = "rpds_py-0.27.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4ed2e16abbc982a169d30d1a420274a709949e2cbdef119fe2ec9d870b42f274"}, + {file = "rpds_py-0.27.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a75f305c9b013289121ec0f1181931975df78738cdf650093e6b86d74aa7d8dd"}, + {file = "rpds_py-0.27.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:67ce7620704745881a3d4b0ada80ab4d99df390838839921f99e63c474f82cf2"}, + {file = "rpds_py-0.27.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9d992ac10eb86d9b6f369647b6a3f412fc0075cfd5d799530e84d335e440a002"}, + {file = "rpds_py-0.27.1-cp313-cp313-win32.whl", hash = "sha256:4f75e4bd8ab8db624e02c8e2fc4063021b58becdbe6df793a8111d9343aec1e3"}, + {file = "rpds_py-0.27.1-cp313-cp313-win_amd64.whl", hash = "sha256:f9025faafc62ed0b75a53e541895ca272815bec18abe2249ff6501c8f2e12b83"}, + {file = "rpds_py-0.27.1-cp313-cp313-win_arm64.whl", hash = "sha256:ed10dc32829e7d222b7d3b93136d25a406ba9788f6a7ebf6809092da1f4d279d"}, + {file = "rpds_py-0.27.1-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:92022bbbad0d4426e616815b16bc4127f83c9a74940e1ccf3cfe0b387aba0228"}, + {file = "rpds_py-0.27.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:47162fdab9407ec3f160805ac3e154df042e577dd53341745fc7fb3f625e6d92"}, + {file = "rpds_py-0.27.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb89bec23fddc489e5d78b550a7b773557c9ab58b7946154a10a6f7a214a48b2"}, + {file = "rpds_py-0.27.1-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e48af21883ded2b3e9eb48cb7880ad8598b31ab752ff3be6457001d78f416723"}, + {file = "rpds_py-0.27.1-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6f5b7bd8e219ed50299e58551a410b64daafb5017d54bbe822e003856f06a802"}, + {file = "rpds_py-0.27.1-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:08f1e20bccf73b08d12d804d6e1c22ca5530e71659e6673bce31a6bb71c1e73f"}, + {file = "rpds_py-0.27.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0dc5dceeaefcc96dc192e3a80bbe1d6c410c469e97bdd47494a7d930987f18b2"}, + {file = "rpds_py-0.27.1-cp313-cp313t-manylinux_2_31_riscv64.whl", hash = "sha256:d76f9cc8665acdc0c9177043746775aa7babbf479b5520b78ae4002d889f5c21"}, + {file = "rpds_py-0.27.1-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:134fae0e36022edad8290a6661edf40c023562964efea0cc0ec7f5d392d2aaef"}, + {file = "rpds_py-0.27.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:eb11a4f1b2b63337cfd3b4d110af778a59aae51c81d195768e353d8b52f88081"}, + {file = "rpds_py-0.27.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:13e608ac9f50a0ed4faec0e90ece76ae33b34c0e8656e3dceb9a7db994c692cd"}, + {file = "rpds_py-0.27.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:dd2135527aa40f061350c3f8f89da2644de26cd73e4de458e79606384f4f68e7"}, + {file = "rpds_py-0.27.1-cp313-cp313t-win32.whl", hash = "sha256:3020724ade63fe320a972e2ffd93b5623227e684315adce194941167fee02688"}, + {file = "rpds_py-0.27.1-cp313-cp313t-win_amd64.whl", hash = "sha256:8ee50c3e41739886606388ba3ab3ee2aae9f35fb23f833091833255a31740797"}, + {file = "rpds_py-0.27.1-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:acb9aafccaae278f449d9c713b64a9e68662e7799dbd5859e2c6b3c67b56d334"}, + {file = "rpds_py-0.27.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:b7fb801aa7f845ddf601c49630deeeccde7ce10065561d92729bfe81bd21fb33"}, + {file = "rpds_py-0.27.1-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fe0dd05afb46597b9a2e11c351e5e4283c741237e7f617ffb3252780cca9336a"}, + {file = "rpds_py-0.27.1-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b6dfb0e058adb12d8b1d1b25f686e94ffa65d9995a5157afe99743bf7369d62b"}, + {file = "rpds_py-0.27.1-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ed090ccd235f6fa8bb5861684567f0a83e04f52dfc2e5c05f2e4b1309fcf85e7"}, + {file = "rpds_py-0.27.1-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bf876e79763eecf3e7356f157540d6a093cef395b65514f17a356f62af6cc136"}, + {file = "rpds_py-0.27.1-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:12ed005216a51b1d6e2b02a7bd31885fe317e45897de81d86dcce7d74618ffff"}, + {file = "rpds_py-0.27.1-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:ee4308f409a40e50593c7e3bb8cbe0b4d4c66d1674a316324f0c2f5383b486f9"}, + {file = "rpds_py-0.27.1-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:0b08d152555acf1f455154d498ca855618c1378ec810646fcd7c76416ac6dc60"}, + {file = "rpds_py-0.27.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:dce51c828941973a5684d458214d3a36fcd28da3e1875d659388f4f9f12cc33e"}, + {file = "rpds_py-0.27.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:c1476d6f29eb81aa4151c9a31219b03f1f798dc43d8af1250a870735516a1212"}, + {file = "rpds_py-0.27.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:3ce0cac322b0d69b63c9cdb895ee1b65805ec9ffad37639f291dd79467bee675"}, + {file = "rpds_py-0.27.1-cp314-cp314-win32.whl", hash = "sha256:dfbfac137d2a3d0725758cd141f878bf4329ba25e34979797c89474a89a8a3a3"}, + {file = "rpds_py-0.27.1-cp314-cp314-win_amd64.whl", hash = "sha256:a6e57b0abfe7cc513450fcf529eb486b6e4d3f8aee83e92eb5f1ef848218d456"}, + {file = "rpds_py-0.27.1-cp314-cp314-win_arm64.whl", hash = "sha256:faf8d146f3d476abfee026c4ae3bdd9ca14236ae4e4c310cbd1cf75ba33d24a3"}, + {file = "rpds_py-0.27.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:ba81d2b56b6d4911ce735aad0a1d4495e808b8ee4dc58715998741a26874e7c2"}, + {file = "rpds_py-0.27.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:84f7d509870098de0e864cad0102711c1e24e9b1a50ee713b65928adb22269e4"}, + {file = "rpds_py-0.27.1-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a9e960fc78fecd1100539f14132425e1d5fe44ecb9239f8f27f079962021523e"}, + {file = "rpds_py-0.27.1-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:62f85b665cedab1a503747617393573995dac4600ff51869d69ad2f39eb5e817"}, + {file = "rpds_py-0.27.1-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fed467af29776f6556250c9ed85ea5a4dd121ab56a5f8b206e3e7a4c551e48ec"}, + {file = "rpds_py-0.27.1-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f2729615f9d430af0ae6b36cf042cb55c0936408d543fb691e1a9e36648fd35a"}, + {file = "rpds_py-0.27.1-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1b207d881a9aef7ba753d69c123a35d96ca7cb808056998f6b9e8747321f03b8"}, + {file = "rpds_py-0.27.1-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:639fd5efec029f99b79ae47e5d7e00ad8a773da899b6309f6786ecaf22948c48"}, + {file = "rpds_py-0.27.1-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fecc80cb2a90e28af8a9b366edacf33d7a91cbfe4c2c4544ea1246e949cfebeb"}, + {file = "rpds_py-0.27.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:42a89282d711711d0a62d6f57d81aa43a1368686c45bc1c46b7f079d55692734"}, + {file = "rpds_py-0.27.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:cf9931f14223de59551ab9d38ed18d92f14f055a5f78c1d8ad6493f735021bbb"}, + {file = "rpds_py-0.27.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:f39f58a27cc6e59f432b568ed8429c7e1641324fbe38131de852cd77b2d534b0"}, + {file = "rpds_py-0.27.1-cp314-cp314t-win32.whl", hash = "sha256:d5fa0ee122dc09e23607a28e6d7b150da16c662e66409bbe85230e4c85bb528a"}, + {file = "rpds_py-0.27.1-cp314-cp314t-win_amd64.whl", hash = "sha256:6567d2bb951e21232c2f660c24cf3470bb96de56cdcb3f071a83feeaff8a2772"}, + {file = "rpds_py-0.27.1-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:c918c65ec2e42c2a78d19f18c553d77319119bf43aa9e2edf7fb78d624355527"}, + {file = "rpds_py-0.27.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1fea2b1a922c47c51fd07d656324531adc787e415c8b116530a1d29c0516c62d"}, + {file = "rpds_py-0.27.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bbf94c58e8e0cd6b6f38d8de67acae41b3a515c26169366ab58bdca4a6883bb8"}, + {file = "rpds_py-0.27.1-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c2a8fed130ce946d5c585eddc7c8eeef0051f58ac80a8ee43bd17835c144c2cc"}, + {file = "rpds_py-0.27.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:037a2361db72ee98d829bc2c5b7cc55598ae0a5e0ec1823a56ea99374cfd73c1"}, + {file = "rpds_py-0.27.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5281ed1cc1d49882f9997981c88df1a22e140ab41df19071222f7e5fc4e72125"}, + {file = "rpds_py-0.27.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2fd50659a069c15eef8aa3d64bbef0d69fd27bb4a50c9ab4f17f83a16cbf8905"}, + {file = "rpds_py-0.27.1-cp39-cp39-manylinux_2_31_riscv64.whl", hash = "sha256:c4b676c4ae3921649a15d28ed10025548e9b561ded473aa413af749503c6737e"}, + {file = "rpds_py-0.27.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:079bc583a26db831a985c5257797b2b5d3affb0386e7ff886256762f82113b5e"}, + {file = "rpds_py-0.27.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:4e44099bd522cba71a2c6b97f68e19f40e7d85399de899d66cdb67b32d7cb786"}, + {file = "rpds_py-0.27.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:e202e6d4188e53c6661af813b46c37ca2c45e497fc558bacc1a7630ec2695aec"}, + {file = "rpds_py-0.27.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:f41f814b8eaa48768d1bb551591f6ba45f87ac76899453e8ccd41dba1289b04b"}, + {file = "rpds_py-0.27.1-cp39-cp39-win32.whl", hash = "sha256:9e71f5a087ead99563c11fdaceee83ee982fd39cf67601f4fd66cb386336ee52"}, + {file = "rpds_py-0.27.1-cp39-cp39-win_amd64.whl", hash = "sha256:71108900c9c3c8590697244b9519017a400d9ba26a36c48381b3f64743a44aab"}, + {file = "rpds_py-0.27.1-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:7ba22cb9693df986033b91ae1d7a979bc399237d45fccf875b76f62bb9e52ddf"}, + {file = "rpds_py-0.27.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:5b640501be9288c77738b5492b3fd3abc4ba95c50c2e41273c8a1459f08298d3"}, + {file = "rpds_py-0.27.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb08b65b93e0c6dd70aac7f7890a9c0938d5ec71d5cb32d45cf844fb8ae47636"}, + {file = "rpds_py-0.27.1-pp310-pypy310_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d7ff07d696a7a38152ebdb8212ca9e5baab56656749f3d6004b34ab726b550b8"}, + {file = "rpds_py-0.27.1-pp310-pypy310_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fb7c72262deae25366e3b6c0c0ba46007967aea15d1eea746e44ddba8ec58dcc"}, + {file = "rpds_py-0.27.1-pp310-pypy310_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7b002cab05d6339716b03a4a3a2ce26737f6231d7b523f339fa061d53368c9d8"}, + {file = "rpds_py-0.27.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:23f6b69d1c26c4704fec01311963a41d7de3ee0570a84ebde4d544e5a1859ffc"}, + {file = "rpds_py-0.27.1-pp310-pypy310_pp73-manylinux_2_31_riscv64.whl", hash = "sha256:530064db9146b247351f2a0250b8f00b289accea4596a033e94be2389977de71"}, + {file = "rpds_py-0.27.1-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7b90b0496570bd6b0321724a330d8b545827c4df2034b6ddfc5f5275f55da2ad"}, + {file = "rpds_py-0.27.1-pp310-pypy310_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:879b0e14a2da6a1102a3fc8af580fc1ead37e6d6692a781bd8c83da37429b5ab"}, + {file = "rpds_py-0.27.1-pp310-pypy310_pp73-musllinux_1_2_i686.whl", hash = "sha256:0d807710df3b5faa66c731afa162ea29717ab3be17bdc15f90f2d9f183da4059"}, + {file = "rpds_py-0.27.1-pp310-pypy310_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:3adc388fc3afb6540aec081fa59e6e0d3908722771aa1e37ffe22b220a436f0b"}, + {file = "rpds_py-0.27.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:c796c0c1cc68cb08b0284db4229f5af76168172670c74908fdbd4b7d7f515819"}, + {file = "rpds_py-0.27.1-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:cdfe4bb2f9fe7458b7453ad3c33e726d6d1c7c0a72960bcc23800d77384e42df"}, + {file = "rpds_py-0.27.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:8fabb8fd848a5f75a2324e4a84501ee3a5e3c78d8603f83475441866e60b94a3"}, + {file = "rpds_py-0.27.1-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eda8719d598f2f7f3e0f885cba8646644b55a187762bec091fa14a2b819746a9"}, + {file = "rpds_py-0.27.1-pp311-pypy311_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3c64d07e95606ec402a0a1c511fe003873fa6af630bda59bac77fac8b4318ebc"}, + {file = "rpds_py-0.27.1-pp311-pypy311_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:93a2ed40de81bcff59aabebb626562d48332f3d028ca2036f1d23cbb52750be4"}, + {file = "rpds_py-0.27.1-pp311-pypy311_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:387ce8c44ae94e0ec50532d9cb0edce17311024c9794eb196b90e1058aadeb66"}, + {file = "rpds_py-0.27.1-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aaf94f812c95b5e60ebaf8bfb1898a7d7cb9c1af5744d4a67fa47796e0465d4e"}, + {file = "rpds_py-0.27.1-pp311-pypy311_pp73-manylinux_2_31_riscv64.whl", hash = "sha256:4848ca84d6ded9b58e474dfdbad4b8bfb450344c0551ddc8d958bf4b36aa837c"}, + {file = "rpds_py-0.27.1-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2bde09cbcf2248b73c7c323be49b280180ff39fadcfe04e7b6f54a678d02a7cf"}, + {file = "rpds_py-0.27.1-pp311-pypy311_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:94c44ee01fd21c9058f124d2d4f0c9dc7634bec93cd4b38eefc385dabe71acbf"}, + {file = "rpds_py-0.27.1-pp311-pypy311_pp73-musllinux_1_2_i686.whl", hash = "sha256:df8b74962e35c9249425d90144e721eed198e6555a0e22a563d29fe4486b51f6"}, + {file = "rpds_py-0.27.1-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:dc23e6820e3b40847e2f4a7726462ba0cf53089512abe9ee16318c366494c17a"}, + {file = "rpds_py-0.27.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:aa8933159edc50be265ed22b401125c9eebff3171f570258854dbce3ecd55475"}, + {file = "rpds_py-0.27.1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:a50431bf02583e21bf273c71b89d710e7a710ad5e39c725b14e685610555926f"}, + {file = "rpds_py-0.27.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:78af06ddc7fe5cc0e967085a9115accee665fb912c22a3f54bad70cc65b05fe6"}, + {file = "rpds_py-0.27.1-pp39-pypy39_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:70d0738ef8fee13c003b100c2fbd667ec4f133468109b3472d249231108283a3"}, + {file = "rpds_py-0.27.1-pp39-pypy39_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e2f6fd8a1cea5bbe599b6e78a6e5ee08db434fc8ffea51ff201c8765679698b3"}, + {file = "rpds_py-0.27.1-pp39-pypy39_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8177002868d1426305bb5de1e138161c2ec9eb2d939be38291d7c431c4712df8"}, + {file = "rpds_py-0.27.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:008b839781d6c9bf3b6a8984d1d8e56f0ec46dc56df61fd669c49b58ae800400"}, + {file = "rpds_py-0.27.1-pp39-pypy39_pp73-manylinux_2_31_riscv64.whl", hash = "sha256:a55b9132bb1ade6c734ddd2759c8dc132aa63687d259e725221f106b83a0e485"}, + {file = "rpds_py-0.27.1-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a46fdec0083a26415f11d5f236b79fa1291c32aaa4a17684d82f7017a1f818b1"}, + {file = "rpds_py-0.27.1-pp39-pypy39_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:8a63b640a7845f2bdd232eb0d0a4a2dd939bcdd6c57e6bb134526487f3160ec5"}, + {file = "rpds_py-0.27.1-pp39-pypy39_pp73-musllinux_1_2_i686.whl", hash = "sha256:7e32721e5d4922deaaf963469d795d5bde6093207c52fec719bd22e5d1bedbc4"}, + {file = "rpds_py-0.27.1-pp39-pypy39_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:2c426b99a068601b5f4623573df7a7c3d72e87533a2dd2253353a03e7502566c"}, + {file = "rpds_py-0.27.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:4fc9b7fe29478824361ead6e14e4f5aed570d477e06088826537e202d25fe859"}, + {file = "rpds_py-0.27.1.tar.gz", hash = "sha256:26a1c73171d10b7acccbded82bf6a586ab8203601e565badc74bbbf8bc5a10f8"}, +] + +[[package]] +name = "ruff" +version = "0.11.7" +description = "An extremely fast Python linter and code formatter, written in Rust." +optional = false +python-versions = ">=3.7" +groups = ["dev"] +files = [ + {file = "ruff-0.11.7-py3-none-linux_armv6l.whl", hash = "sha256:d29e909d9a8d02f928d72ab7837b5cbc450a5bdf578ab9ebee3263d0a525091c"}, + {file = "ruff-0.11.7-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:dd1fb86b168ae349fb01dd497d83537b2c5541fe0626e70c786427dd8363aaee"}, + {file = "ruff-0.11.7-py3-none-macosx_11_0_arm64.whl", hash = "sha256:d3d7d2e140a6fbbc09033bce65bd7ea29d6a0adeb90b8430262fbacd58c38ada"}, + {file = "ruff-0.11.7-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4809df77de390a1c2077d9b7945d82f44b95d19ceccf0c287c56e4dc9b91ca64"}, + {file = "ruff-0.11.7-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f3a0c2e169e6b545f8e2dba185eabbd9db4f08880032e75aa0e285a6d3f48201"}, + {file = "ruff-0.11.7-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:49b888200a320dd96a68e86736cf531d6afba03e4f6cf098401406a257fcf3d6"}, + {file = "ruff-0.11.7-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:2b19cdb9cf7dae00d5ee2e7c013540cdc3b31c4f281f1dacb5a799d610e90db4"}, + {file = "ruff-0.11.7-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:64e0ee994c9e326b43539d133a36a455dbaab477bc84fe7bfbd528abe2f05c1e"}, + {file = "ruff-0.11.7-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bad82052311479a5865f52c76ecee5d468a58ba44fb23ee15079f17dd4c8fd63"}, + {file = "ruff-0.11.7-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7940665e74e7b65d427b82bffc1e46710ec7f30d58b4b2d5016e3f0321436502"}, + {file = "ruff-0.11.7-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:169027e31c52c0e36c44ae9a9c7db35e505fee0b39f8d9fca7274a6305295a92"}, + {file = "ruff-0.11.7-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:305b93f9798aee582e91e34437810439acb28b5fc1fee6b8205c78c806845a94"}, + {file = "ruff-0.11.7-py3-none-musllinux_1_2_i686.whl", hash = "sha256:a681db041ef55550c371f9cd52a3cf17a0da4c75d6bd691092dfc38170ebc4b6"}, + {file = "ruff-0.11.7-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:07f1496ad00a4a139f4de220b0c97da6d4c85e0e4aa9b2624167b7d4d44fd6b6"}, + {file = "ruff-0.11.7-py3-none-win32.whl", hash = "sha256:f25dfb853ad217e6e5f1924ae8a5b3f6709051a13e9dad18690de6c8ff299e26"}, + {file = "ruff-0.11.7-py3-none-win_amd64.whl", hash = "sha256:0a931d85959ceb77e92aea4bbedfded0a31534ce191252721128f77e5ae1f98a"}, + {file = "ruff-0.11.7-py3-none-win_arm64.whl", hash = "sha256:778c1e5d6f9e91034142dfd06110534ca13220bfaad5c3735f6cb844654f6177"}, + {file = "ruff-0.11.7.tar.gz", hash = "sha256:655089ad3224070736dc32844fde783454f8558e71f501cb207485fe4eee23d4"}, +] [[package]] name = "setuptools" -version = "67.7.2" +version = "80.9.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" -category = "dev" optional = false -python-versions = ">=3.7" +python-versions = ">=3.9" +groups = ["executable"] +markers = "python_version < \"3.15\"" files = [ - {file = "setuptools-67.7.2-py3-none-any.whl", hash = "sha256:23aaf86b85ca52ceb801d32703f12d77517b2556af839621c641fca11287952b"}, - {file = "setuptools-67.7.2.tar.gz", hash = "sha256:f104fa03692a2602fa0fec6c6a9e63b6c8a968de13e17c026957dd1f53d80990"}, + {file = "setuptools-80.9.0-py3-none-any.whl", hash = "sha256:062d34222ad13e0cc312a4c02d73f059e86a4acbfbdea8f8f76b28c99f306922"}, + {file = "setuptools-80.9.0.tar.gz", hash = "sha256:f36b47402ecde768dbfafc46e8e4207b4360c654f1f3bb84475f0a28628fb19c"}, ] [package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] -testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\"", "ruff (>=0.8.0) ; sys_platform != \"cygwin\""] +core = ["importlib_metadata (>=6) ; python_version < \"3.10\"", "jaraco.functools (>=4)", "jaraco.text (>=3.7)", "more_itertools", "more_itertools (>=8.8)", "packaging (>=24.2)", "platformdirs (>=4.2.2)", "tomli (>=2.0.1) ; python_version < \"3.11\"", "wheel (>=0.43.0)"] +cover = ["pytest-cov"] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier", "towncrier (<24.7)"] +enabler = ["pytest-enabler (>=2.2)"] +test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21) ; python_version >= \"3.9\" and sys_platform != \"cygwin\"", "jaraco.envs (>=2.2)", "jaraco.path (>=3.7.2)", "jaraco.test (>=5.5)", "packaging (>=24.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf ; sys_platform != \"cygwin\"", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] +type = ["importlib_metadata (>=7.0.2) ; python_version < \"3.10\"", "jaraco.develop (>=7.21) ; sys_platform != \"cygwin\"", "mypy (==1.14.*)", "pytest-mypy"] + +[[package]] +name = "shellingham" +version = "1.5.4" +description = "Tool to Detect Surrounding Shell" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686"}, + {file = "shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de"}, +] [[package]] name = "six" -version = "1.16.0" +version = "1.17.0" description = "Python 2 and 3 compatibility utilities" -category = "main" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +groups = ["main"] files = [ - {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, - {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, + {file = "six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274"}, + {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"}, ] [[package]] name = "smmap" -version = "5.0.0" +version = "5.0.2" description = "A pure Python implementation of a sliding window memory map manager" -category = "main" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" +groups = ["main"] files = [ - {file = "smmap-5.0.0-py3-none-any.whl", hash = "sha256:2aba19d6a040e78d8b09de5c57e96207b09ed71d8e55ce0959eeee6c8e190d94"}, - {file = "smmap-5.0.0.tar.gz", hash = "sha256:c840e62059cd3be204b0c9c9f74be2c09d5648eddd4580d9314c3ecde0b30936"}, + {file = "smmap-5.0.2-py3-none-any.whl", hash = "sha256:b30115f0def7d7531d22a0fb6502488d879e75b260a9db4d0819cfb25403af5e"}, + {file = "smmap-5.0.2.tar.gz", hash = "sha256:26ea65a03958fa0c8a1c7e8c7a58fdc77221b8910f6be2131affade476898ad5"}, ] [[package]] -name = "spinners" -version = "0.0.24" -description = "Spinners for terminals" -category = "main" +name = "sniffio" +version = "1.3.1" +description = "Sniff out which async library your code is running under" optional = false -python-versions = "*" +python-versions = ">=3.7" +groups = ["main"] +markers = "python_version >= \"3.10\"" files = [ - {file = "spinners-0.0.24-py3-none-any.whl", hash = "sha256:2fa30d0b72c9650ad12bbe031c9943b8d441e41b4f5602b0ec977a19f3290e98"}, - {file = "spinners-0.0.24.tar.gz", hash = "sha256:1eb6aeb4781d72ab42ed8a01dcf20f3002bf50740d7154d12fb8c9769bf9e27f"}, + {file = "sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2"}, + {file = "sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc"}, ] [[package]] -name = "termcolor" -version = "2.3.0" -description = "ANSI color formatting for output in terminal" -category = "main" +name = "sse-starlette" +version = "3.0.2" +description = "SSE plugin for Starlette" optional = false -python-versions = ">=3.7" +python-versions = ">=3.9" +groups = ["main"] +markers = "python_version >= \"3.10\"" files = [ - {file = "termcolor-2.3.0-py3-none-any.whl", hash = "sha256:3afb05607b89aed0ffe25202399ee0867ad4d3cb4180d98aaf8eefa6a5f7d475"}, - {file = "termcolor-2.3.0.tar.gz", hash = "sha256:b5b08f68937f138fe92f6c089b99f1e2da0ae56c52b78bf7075fd95420fd9a5a"}, + {file = "sse_starlette-3.0.2-py3-none-any.whl", hash = "sha256:16b7cbfddbcd4eaca11f7b586f3b8a080f1afe952c15813455b162edea619e5a"}, + {file = "sse_starlette-3.0.2.tar.gz", hash = "sha256:ccd60b5765ebb3584d0de2d7a6e4f745672581de4f5005ab31c3a25d10b52b3a"}, ] +[package.dependencies] +anyio = ">=4.7.0" + [package.extras] -tests = ["pytest", "pytest-cov"] +daphne = ["daphne (>=4.2.0)"] +examples = ["aiosqlite (>=0.21.0)", "fastapi (>=0.115.12)", "sqlalchemy[asyncio] (>=2.0.41)", "starlette (>=0.41.3)", "uvicorn (>=0.34.0)"] +granian = ["granian (>=2.3.1)"] +uvicorn = ["uvicorn (>=0.34.0)"] [[package]] -name = "texttable" -version = "1.6.7" -description = "module to create simple ASCII tables" -category = "main" +name = "starlette" +version = "0.49.1" +description = "The little ASGI library that shines." optional = false -python-versions = "*" +python-versions = ">=3.9" +groups = ["main"] +markers = "python_version >= \"3.10\"" files = [ - {file = "texttable-1.6.7-py2.py3-none-any.whl", hash = "sha256:b7b68139aa8a6339d2c320ca8b1dc42d13a7831a346b446cb9eb385f0c76310c"}, - {file = "texttable-1.6.7.tar.gz", hash = "sha256:290348fb67f7746931bcdfd55ac7584ecd4e5b0846ab164333f0794b121760f2"}, + {file = "starlette-0.49.1-py3-none-any.whl", hash = "sha256:d92ce9f07e4a3caa3ac13a79523bd18e3bc0042bb8ff2d759a8e7dd0e1859875"}, + {file = "starlette-0.49.1.tar.gz", hash = "sha256:481a43b71e24ed8c43b11ea02f5353d77840e01480881b8cb5a26b8cae64a8cb"}, ] +[package.dependencies] +anyio = ">=3.6.2,<5" +typing-extensions = {version = ">=4.10.0", markers = "python_version < \"3.13\""} + +[package.extras] +full = ["httpx (>=0.27.0,<0.29.0)", "itsdangerous", "jinja2", "python-multipart (>=0.0.18)", "pyyaml"] + +[[package]] +name = "tenacity" +version = "9.0.0" +description = "Retry code until it succeeds" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "tenacity-9.0.0-py3-none-any.whl", hash = "sha256:93de0c98785b27fcf659856aa9f54bfbd399e29969b0621bc7f762bd441b4539"}, + {file = "tenacity-9.0.0.tar.gz", hash = "sha256:807f37ca97d62aa361264d497b0e31e92b8027044942bfa756160d908320d73b"}, +] + +[package.extras] +doc = ["reno", "sphinx"] +test = ["pytest", "tornado (>=4.5)", "typeguard"] + [[package]] name = "tomli" -version = "2.0.1" +version = "2.3.0" description = "A lil' TOML parser" -category = "dev" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" +groups = ["test"] +markers = "python_version < \"3.11\"" files = [ - {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, - {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, + {file = "tomli-2.3.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:88bd15eb972f3664f5ed4b57c1634a97153b4bac4479dcb6a495f41921eb7f45"}, + {file = "tomli-2.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:883b1c0d6398a6a9d29b508c331fa56adbcdff647f6ace4dfca0f50e90dfd0ba"}, + {file = "tomli-2.3.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d1381caf13ab9f300e30dd8feadb3de072aeb86f1d34a8569453ff32a7dea4bf"}, + {file = "tomli-2.3.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a0e285d2649b78c0d9027570d4da3425bdb49830a6156121360b3f8511ea3441"}, + {file = "tomli-2.3.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:0a154a9ae14bfcf5d8917a59b51ffd5a3ac1fd149b71b47a3a104ca4edcfa845"}, + {file = "tomli-2.3.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:74bf8464ff93e413514fefd2be591c3b0b23231a77f901db1eb30d6f712fc42c"}, + {file = "tomli-2.3.0-cp311-cp311-win32.whl", hash = "sha256:00b5f5d95bbfc7d12f91ad8c593a1659b6387b43f054104cda404be6bda62456"}, + {file = "tomli-2.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:4dc4ce8483a5d429ab602f111a93a6ab1ed425eae3122032db7e9acf449451be"}, + {file = "tomli-2.3.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d7d86942e56ded512a594786a5ba0a5e521d02529b3826e7761a05138341a2ac"}, + {file = "tomli-2.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:73ee0b47d4dad1c5e996e3cd33b8a76a50167ae5f96a2607cbe8cc773506ab22"}, + {file = "tomli-2.3.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:792262b94d5d0a466afb5bc63c7daa9d75520110971ee269152083270998316f"}, + {file = "tomli-2.3.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4f195fe57ecceac95a66a75ac24d9d5fbc98ef0962e09b2eddec5d39375aae52"}, + {file = "tomli-2.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e31d432427dcbf4d86958c184b9bfd1e96b5b71f8eb17e6d02531f434fd335b8"}, + {file = "tomli-2.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:7b0882799624980785240ab732537fcfc372601015c00f7fc367c55308c186f6"}, + {file = "tomli-2.3.0-cp312-cp312-win32.whl", hash = "sha256:ff72b71b5d10d22ecb084d345fc26f42b5143c5533db5e2eaba7d2d335358876"}, + {file = "tomli-2.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:1cb4ed918939151a03f33d4242ccd0aa5f11b3547d0cf30f7c74a408a5b99878"}, + {file = "tomli-2.3.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5192f562738228945d7b13d4930baffda67b69425a7f0da96d360b0a3888136b"}, + {file = "tomli-2.3.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:be71c93a63d738597996be9528f4abe628d1adf5e6eb11607bc8fe1a510b5dae"}, + {file = "tomli-2.3.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c4665508bcbac83a31ff8ab08f424b665200c0e1e645d2bd9ab3d3e557b6185b"}, + {file = "tomli-2.3.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4021923f97266babc6ccab9f5068642a0095faa0a51a246a6a02fccbb3514eaf"}, + {file = "tomli-2.3.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a4ea38c40145a357d513bffad0ed869f13c1773716cf71ccaa83b0fa0cc4e42f"}, + {file = "tomli-2.3.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ad805ea85eda330dbad64c7ea7a4556259665bdf9d2672f5dccc740eb9d3ca05"}, + {file = "tomli-2.3.0-cp313-cp313-win32.whl", hash = "sha256:97d5eec30149fd3294270e889b4234023f2c69747e555a27bd708828353ab606"}, + {file = "tomli-2.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:0c95ca56fbe89e065c6ead5b593ee64b84a26fca063b5d71a1122bf26e533999"}, + {file = "tomli-2.3.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:cebc6fe843e0733ee827a282aca4999b596241195f43b4cc371d64fc6639da9e"}, + {file = "tomli-2.3.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4c2ef0244c75aba9355561272009d934953817c49f47d768070c3c94355c2aa3"}, + {file = "tomli-2.3.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c22a8bf253bacc0cf11f35ad9808b6cb75ada2631c2d97c971122583b129afbc"}, + {file = "tomli-2.3.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0eea8cc5c5e9f89c9b90c4896a8deefc74f518db5927d0e0e8d4a80953d774d0"}, + {file = "tomli-2.3.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:b74a0e59ec5d15127acdabd75ea17726ac4c5178ae51b85bfe39c4f8a278e879"}, + {file = "tomli-2.3.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:b5870b50c9db823c595983571d1296a6ff3e1b88f734a4c8f6fc6188397de005"}, + {file = "tomli-2.3.0-cp314-cp314-win32.whl", hash = "sha256:feb0dacc61170ed7ab602d3d972a58f14ee3ee60494292d384649a3dc38ef463"}, + {file = "tomli-2.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:b273fcbd7fc64dc3600c098e39136522650c49bca95df2d11cf3b626422392c8"}, + {file = "tomli-2.3.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:940d56ee0410fa17ee1f12b817b37a4d4e4dc4d27340863cc67236c74f582e77"}, + {file = "tomli-2.3.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f85209946d1fe94416debbb88d00eb92ce9cd5266775424ff81bc959e001acaf"}, + {file = "tomli-2.3.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a56212bdcce682e56b0aaf79e869ba5d15a6163f88d5451cbde388d48b13f530"}, + {file = "tomli-2.3.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c5f3ffd1e098dfc032d4d3af5c0ac64f6d286d98bc148698356847b80fa4de1b"}, + {file = "tomli-2.3.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5e01decd096b1530d97d5d85cb4dff4af2d8347bd35686654a004f8dea20fc67"}, + {file = "tomli-2.3.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:8a35dd0e643bb2610f156cca8db95d213a90015c11fee76c946aa62b7ae7e02f"}, + {file = "tomli-2.3.0-cp314-cp314t-win32.whl", hash = "sha256:a1f7f282fe248311650081faafa5f4732bdbfef5d45fe3f2e702fbc6f2d496e0"}, + {file = "tomli-2.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:70a251f8d4ba2d9ac2542eecf008b3c8a9fc5c3f9f02c56a9d7952612be2fdba"}, + {file = "tomli-2.3.0-py3-none-any.whl", hash = "sha256:e95b1af3c5b07d9e643909b5abbec77cd9f1217e6d0bca72b0234736b9fb1f1b"}, + {file = "tomli-2.3.0.tar.gz", hash = "sha256:64be704a875d2a59753d80ee8a533c3fe183e3f06807ff7dc2232938ccb01549"}, ] [[package]] -name = "types-pyyaml" -version = "6.0.12.9" -description = "Typing stubs for PyYAML" -category = "dev" +name = "typer" +version = "0.15.4" +description = "Typer, build great CLIs. Easy to code. Based on Python type hints." optional = false -python-versions = "*" +python-versions = ">=3.7" +groups = ["main"] files = [ - {file = "types-PyYAML-6.0.12.9.tar.gz", hash = "sha256:c51b1bd6d99ddf0aa2884a7a328810ebf70a4262c292195d3f4f9a0005f9eeb6"}, - {file = "types_PyYAML-6.0.12.9-py3-none-any.whl", hash = "sha256:5aed5aa66bd2d2e158f75dda22b059570ede988559f030cf294871d3b647e3e8"}, + {file = "typer-0.15.4-py3-none-any.whl", hash = "sha256:eb0651654dcdea706780c466cf06d8f174405a659ffff8f163cfbfee98c0e173"}, + {file = "typer-0.15.4.tar.gz", hash = "sha256:89507b104f9b6a0730354f27c39fae5b63ccd0c95b1ce1f1a6ba0cfd329997c3"}, ] +[package.dependencies] +click = ">=8.0.0,<8.2" +rich = ">=10.11.0" +shellingham = ">=1.3.0" +typing-extensions = ">=3.7.4.3" + [[package]] name = "typing-extensions" -version = "4.5.0" -description = "Backported and Experimental Type Hints for Python 3.7+" -category = "main" +version = "4.15.0" +description = "Backported and Experimental Type Hints for Python 3.9+" optional = false -python-versions = ">=3.7" +python-versions = ">=3.9" +groups = ["main", "test"] files = [ - {file = "typing_extensions-4.5.0-py3-none-any.whl", hash = "sha256:fb33085c39dd998ac16d1431ebc293a8b3eedd00fd4a32de0ff79002c19511b4"}, - {file = "typing_extensions-4.5.0.tar.gz", hash = "sha256:5cb5f4a79139d699607b3ef622a1dedafa84e115ab0024e0d9c044a9479ca7cb"}, + {file = "typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548"}, + {file = "typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466"}, +] +markers = {test = "python_version < \"3.11\""} + +[[package]] +name = "typing-inspection" +version = "0.4.2" +description = "Runtime typing introspection tools" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7"}, + {file = "typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464"}, +] + +[package.dependencies] +typing-extensions = ">=4.12.0" + +[[package]] +name = "tzdata" +version = "2025.3" +description = "Provider of IANA time zone data" +optional = false +python-versions = ">=2" +groups = ["main"] +files = [ + {file = "tzdata-2025.3-py2.py3-none-any.whl", hash = "sha256:06a47e5700f3081aab02b2e513160914ff0694bce9947d6b76ebd6bf57cfc5d1"}, + {file = "tzdata-2025.3.tar.gz", hash = "sha256:de39c2ca5dc7b0344f2eba86f49d614019d29f060fc4ebc8a417896a620b56a7"}, ] [[package]] name = "urllib3" -version = "2.0.2" +version = "2.6.3" description = "HTTP library with thread-safe connection pooling, file post, and more." -category = "main" optional = false -python-versions = ">=3.7" +python-versions = ">=3.9" +groups = ["main", "test"] files = [ - {file = "urllib3-2.0.2-py3-none-any.whl", hash = "sha256:d055c2f9d38dc53c808f6fdc8eab7360b6fdbbde02340ed25cfbcd817c62469e"}, - {file = "urllib3-2.0.2.tar.gz", hash = "sha256:61717a1095d7e155cdb737ac7bb2f4324a858a1e2e6466f6d03ff630ca68d3cc"}, + {file = "urllib3-2.6.3-py3-none-any.whl", hash = "sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4"}, + {file = "urllib3-2.6.3.tar.gz", hash = "sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed"}, ] [package.extras] -brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] -secure = ["certifi", "cryptography (>=1.9)", "idna (>=2.0.0)", "pyopenssl (>=17.1.0)", "urllib3-secure-extra"] +brotli = ["brotli (>=1.2.0) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=1.2.0.0) ; platform_python_implementation != \"CPython\""] +h2 = ["h2 (>=4,<5)"] socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] -zstd = ["zstandard (>=0.18.0)"] +zstd = ["backports-zstd (>=1.0.0) ; python_version < \"3.14\""] + +[[package]] +name = "uvicorn" +version = "0.38.0" +description = "The lightning-fast ASGI server." +optional = false +python-versions = ">=3.9" +groups = ["main"] +markers = "python_version >= \"3.10\" and sys_platform != \"emscripten\"" +files = [ + {file = "uvicorn-0.38.0-py3-none-any.whl", hash = "sha256:48c0afd214ceb59340075b4a052ea1ee91c16fbc2a9b1469cca0e54566977b02"}, + {file = "uvicorn-0.38.0.tar.gz", hash = "sha256:fd97093bdd120a2609fc0d3afe931d4d4ad688b6e75f0f929fde1bc36fe0e91d"}, +] + +[package.dependencies] +click = ">=7.0" +h11 = ">=0.8" +typing-extensions = {version = ">=4.0", markers = "python_version < \"3.11\""} + +[package.extras] +standard = ["colorama (>=0.4) ; sys_platform == \"win32\"", "httptools (>=0.6.3)", "python-dotenv (>=0.13)", "pyyaml (>=5.1)", "uvloop (>=0.15.1) ; sys_platform != \"win32\" and sys_platform != \"cygwin\" and platform_python_implementation != \"PyPy\"", "watchfiles (>=0.13)", "websockets (>=10.4)"] [[package]] name = "zipp" -version = "3.15.0" +version = "3.23.0" description = "Backport of pathlib-compatible object wrapper for zip files" -category = "main" optional = false -python-versions = ">=3.7" +python-versions = ">=3.9" +groups = ["executable"] +markers = "python_version == \"3.9\"" files = [ - {file = "zipp-3.15.0-py3-none-any.whl", hash = "sha256:48904fc76a60e542af151aded95726c1a5c34ed43ab4134b597665c86d7ad556"}, - {file = "zipp-3.15.0.tar.gz", hash = "sha256:112929ad649da941c23de50f356a2b5570c954b65150642bccdd66bf194d224b"}, + {file = "zipp-3.23.0-py3-none-any.whl", hash = "sha256:071652d6115ed432f5ce1d34c336c0adfd6a884660d1e9712a256d3d3bd4b14e"}, + {file = "zipp-3.23.0.tar.gz", hash = "sha256:a07157588a12518c9d4034df3fbbee09c814741a33ff63c05fa29d26a2404166"}, ] [package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\""] +cover = ["pytest-cov"] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +enabler = ["pytest-enabler (>=2.2)"] +test = ["big-O", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more_itertools", "pytest (>=6,!=8.1.*)", "pytest-ignore-flaky"] +type = ["pytest-mypy"] [metadata] -lock-version = "2.0" -python-versions = ">=3.7,<3.12" -content-hash = "ab665eabb7b281d5fcfef66b81ab5882b511282e822129e2408b1653f5044dd5" +lock-version = "2.1" +python-versions = ">=3.9" +content-hash = "04201585f115c406a49b035b4c3b3be7057baee685997ab57fe39cc964ad5352" diff --git a/process_executable_file.py b/process_executable_file.py new file mode 100755 index 00000000..36d6d0d6 --- /dev/null +++ b/process_executable_file.py @@ -0,0 +1,187 @@ +#!/usr/bin/env python3 + +"""Used in the GitHub Actions workflow (build_executable.yml) to process the executable file. + +This script calculates hash and renames executable file depending on the OS, arch, and build mode. +It also creates a file with the hash of the executable file. +It uses SHA256 algorithm to calculate the hash. +It returns the name of the executable file which is used as artifact name. +""" + +import argparse +import hashlib +import os +import platform +import shutil +from pathlib import Path +from string import Template +from typing import Union + +_ARCHIVE_FORMAT = 'zip' +_HASH_FILE_EXT = '.sha256' +_OS_TO_CLI_DIST_TEMPLATE = { + 'darwin': Template('cycode-mac$suffix$ext'), + 'linux': Template('cycode-linux$suffix$ext'), + 'windows': Template('cycode-win$suffix.exe$ext'), +} +_WINDOWS = 'windows' +_WINDOWS_EXECUTABLE_SUFFIX = '.exe' + +DirHashes = list[tuple[str, str]] + + +def get_hash_of_file(file_path: Union[str, Path]) -> str: + with open(file_path, 'rb') as f: + return hashlib.sha256(f.read()).hexdigest() + + +def get_hashes_of_many_files(root: str, file_paths: list[str]) -> DirHashes: + hashes = [] + + for file_path in file_paths: + file_path = os.path.join(root, file_path) + file_hash = get_hash_of_file(file_path) + + hashes.append((file_hash, file_path)) + + return hashes + + +def get_hashes_of_every_file_in_the_directory(dir_path: Path) -> DirHashes: + hashes = [] + + for root, _, files in os.walk(dir_path): + hashes.extend( + get_hashes_of_many_files( + root, + files, + ) + ) + + return hashes + + +def normalize_hashes_db(hashes: DirHashes, dir_path: Path) -> DirHashes: + normalized_hashes = [] + + for file_hash, file_path in hashes: + relative_file_path = file_path[file_path.find(dir_path.name) :] + normalized_hashes.append((file_hash, relative_file_path)) + + # sort by file path + normalized_hashes.sort(key=lambda hash_item: hash_item[1]) + + return normalized_hashes + + +def is_arm() -> bool: + return platform.machine().lower() in ('arm', 'arm64', 'aarch64') + + +def get_os_name() -> str: + return platform.system().lower() + + +def get_cli_file_name(suffix: str = '', ext: str = '') -> str: + os_name = get_os_name() + if os_name not in _OS_TO_CLI_DIST_TEMPLATE: + raise Exception(f'Unsupported OS: {os_name}') + + template = _OS_TO_CLI_DIST_TEMPLATE[os_name] + return template.substitute(suffix=suffix, ext=ext) + + +def get_cli_file_suffix(is_onedir: bool) -> str: + suffixes = [] + + if is_arm(): + suffixes.append('-arm') + if is_onedir: + suffixes.append('-onedir') + + return ''.join(suffixes) + + +def write_hash_to_file(file_hash: str, output_path: str) -> None: + with open(output_path, 'w') as f: + f.write(file_hash) + + +def write_hashes_db_to_file(hashes: DirHashes, output_path: str) -> None: + content = '' + for file_hash, file_path in hashes: + content += f'{file_hash} {file_path}\n' + + with open(output_path, 'w') as f: + f.write(content) + + +def get_cli_filename(is_onedir: bool) -> str: + return get_cli_file_name(get_cli_file_suffix(is_onedir)) + + +def get_cli_path(output_path: Path, is_onedir: bool) -> str: + return os.path.join(output_path, get_cli_filename(is_onedir)) + + +def get_cli_hash_filename(is_onedir: bool) -> str: + return get_cli_file_name(suffix=get_cli_file_suffix(is_onedir), ext=_HASH_FILE_EXT) + + +def get_cli_hash_path(output_path: Path, is_onedir: bool) -> str: + return os.path.join(output_path, get_cli_hash_filename(is_onedir)) + + +def get_cli_archive_filename(is_onedir: bool) -> str: + return get_cli_file_name(suffix=get_cli_file_suffix(is_onedir)) + + +def get_cli_archive_path(output_path: Path, is_onedir: bool) -> str: + return os.path.join(output_path, get_cli_archive_filename(is_onedir)) + + +def archive_directory(input_path: Path, output_path: str) -> None: + shutil.make_archive(output_path.removesuffix(f'.{_ARCHIVE_FORMAT}'), _ARCHIVE_FORMAT, input_path) + + +def process_executable_file(input_path: Path, is_onedir: bool) -> str: + output_path = input_path.parent + hash_file_path = get_cli_hash_path(output_path, is_onedir) + + if is_onedir: + hashes = get_hashes_of_every_file_in_the_directory(input_path) + normalized_hashes = normalize_hashes_db(hashes, input_path) + write_hashes_db_to_file(normalized_hashes, hash_file_path) + + archived_file_path = get_cli_archive_path(output_path, is_onedir) + archive_directory(input_path, f'{archived_file_path}.{_ARCHIVE_FORMAT}') + shutil.rmtree(input_path) + else: + file_hash = get_hash_of_file(input_path) + write_hash_to_file(file_hash, hash_file_path) + + # for example rename cycode-cli to cycode-mac or cycode-mac-arm-onedir + os.rename(input_path, get_cli_path(output_path, is_onedir)) + + return get_cli_filename(is_onedir) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument('input', help='Path to executable or directory') + + args = parser.parse_args() + input_path = Path(args.input) + is_onedir = input_path.is_dir() + + if get_os_name() == _WINDOWS and not is_onedir and input_path.suffix != _WINDOWS_EXECUTABLE_SUFFIX: + # add .exe on windows if was missed (to simplify GHA workflow) + input_path = input_path.with_suffix(_WINDOWS_EXECUTABLE_SUFFIX) + + artifact_name = process_executable_file(input_path, is_onedir) + + print(artifact_name) # noqa: T201 + + +if __name__ == '__main__': + main() diff --git a/pyinstaller.spec b/pyinstaller.spec index 6409682a..c577c547 100644 --- a/pyinstaller.spec +++ b/pyinstaller.spec @@ -1,64 +1,47 @@ # -*- mode: python ; coding: utf-8 -*- # Run `poetry run pyinstaller pyinstaller.spec` to generate the binary. +# Set the env var `CYCODE_ONEDIR_MODE` to generate a single directory instead of a single file. - -block_cipher = None - -INIT_FILE_PATH = os.path.join('cycode', '__init__.py') +_INIT_FILE_PATH = os.path.join('cycode', '__init__.py') +_CODESIGN_IDENTITY = os.environ.get('APPLE_CERT_NAME') +_ONEDIR_MODE = os.environ.get('CYCODE_ONEDIR_MODE') is not None # save the prev content of __init__ file -with open(INIT_FILE_PATH, 'r', encoding='UTF-8') as file: +with open(_INIT_FILE_PATH, 'r', encoding='UTF-8') as file: prev_content = file.read() import dunamai as _dunamai + VERSION_PLACEHOLDER = '0.0.0' CLI_VERSION = _dunamai.get_version('cycode', first_choice=_dunamai.Version.from_git).serialize( metadata=False, bump=True, style=_dunamai.Style.Pep440 ) -# write version from Git Tag to freeze the value and don't depend on Git -with open(INIT_FILE_PATH, 'w', encoding='UTF-8') as file: +# write the version from Git Tag to freeze the value and don't depend on Git +with open(_INIT_FILE_PATH, 'w', encoding='UTF-8') as file: file.write(prev_content.replace(VERSION_PLACEHOLDER, CLI_VERSION)) a = Analysis( - ['cycode/cli/main.py'], - pathex=[], - binaries=[], - datas=[('cycode/cli/config.yaml', 'cycode/cli'), ('cycode/cyclient/config.yaml', 'cycode/cyclient')], - hiddenimports=[], - hookspath=[], - hooksconfig={}, - runtime_hooks=[], - excludes=['tests'], - win_no_prefer_redirects=False, - win_private_assemblies=False, - cipher=block_cipher, - noarchive=False, + scripts=['cycode/cli/main.py'], + excludes=['tests', 'setuptools', 'pkg_resources'], ) -pyz = PYZ(a.pure, a.zipped_data, cipher=block_cipher) + +exe_args = [PYZ(a.pure), a.scripts, a.binaries, a.datas] +if _ONEDIR_MODE: + exe_args = [PYZ(a.pure), a.scripts] exe = EXE( - pyz, - a.scripts, - a.binaries, - a.zipfiles, - a.datas, - [], - name='cycode', - debug=False, - bootloader_ignore_signals=False, - strip=False, - upx=True, - upx_exclude=[], - runtime_tmpdir=None, - console=True, - disable_windowed_traceback=False, - argv_emulation=False, + *exe_args, + name='cycode-cli', + exclude_binaries=bool(_ONEDIR_MODE), target_arch=None, - codesign_identity=None, - entitlements_file=None, + codesign_identity=_CODESIGN_IDENTITY, + entitlements_file='entitlements.plist', ) +if _ONEDIR_MODE: + coll = COLLECT(exe, a.binaries, a.datas, name='cycode-cli') + # rollback the prev content of the __init__ file -with open(INIT_FILE_PATH, 'w', encoding='UTF-8') as file: +with open(_INIT_FILE_PATH, 'w', encoding='UTF-8') as file: file.write(prev_content) diff --git a/pyproject.toml b/pyproject.toml index e95712de..2beebaf2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,54 +1,69 @@ -[tool.poetry] +[project] name = "cycode" -version = "0.0.0" # DON'T TOUCH. Placeholder. Will be filled automatically on poetry build from Git Tag -description = "Perform secrets/iac scans for your sources using Cycode's engine" -keywords=["secret-scan", "cycode", "devops", "token", "secret", "security", "cycode", "code"] -authors = ["Cycode "] +description = "Boost security in your dev lifecycle via SAST, SCA, Secrets & IaC scanning." +keywords = ["secret-scan", "cycode", "devops", "token", "secret", "security", "code"] +authors = [{name = "Cycode", email = "support@cycode.com"}] license = "MIT" -repository = "https://github.com/cycodehq-public/cycode-cli" +requires-python = ">=3.9" readme = "README.md" classifiers = [ "Development Status :: 5 - Production/Stable", "Environment :: Console", - "License :: OSI Approved :: MIT License", "Natural Language :: English", "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", ] +dynamic = ["dependencies", "version"] + +[project.scripts] +cycode = "cycode.cli.app:app" + +[project.urls] +repository = "https://github.com/cycodehq/cycode-cli" -[tool.poetry.scripts] -cycode = "cycode.cli.main:main_cli" +[tool.poetry] +requires-poetry = ">=2.0" +version = "0.0.0" # DON'T TOUCH. Placeholder. Will be filled automatically on poetry build from Git Tag [tool.poetry.dependencies] -python = ">=3.7,<3.12" click = ">=8.1.0,<8.2.0" colorama = ">=0.4.3,<0.5.0" pyyaml = ">=6.0,<7.0" -marshmallow = ">=3.8.0,<3.9.0" -pathspec = ">=0.8.0,<0.9.0" +marshmallow = ">=3.15.0,<4.0.0" gitpython = ">=3.1.30,<3.2.0" -arrow = ">=0.17.0,<0.18.0" -binaryornot = ">=0.4.4,<0.5.0" -halo = "==0.0.31" -texttable = ">=1.6.7,<1.7.0" -requests = ">=2.24,<3.0" +arrow = ">=1.0.0,<1.5.0" +requests = ">=2.32.4,<3.0" +urllib3 = ">=2.4.0,<3.0.0" +pyjwt = ">=2.8.0,<3.0" +rich = ">=13.9.4, <14" +patch-ng = "1.19.0" +typer = "^0.15.3" +tenacity = ">=9.0.0,<9.1.0" +mcp = { version = ">=1.9.3,<2.0.0", markers = "python_version >= '3.10'" } +pydantic = ">=2.11.5,<3.0.0" +pathvalidate = ">=3.3.1,<4.0.0" [tool.poetry.group.test.dependencies] mock = ">=4.0.3,<4.1.0" pytest = ">=7.3.1,<7.4.0" pytest-mock = ">=3.10.0,<3.11.0" coverage = ">=7.2.3,<7.3.0" -responses = ">=0.23.1,<0.24.0" +responses = ">=0.23.1,<0.27.0" +pyfakefs = ">=5.7.2,<5.11.0" [tool.poetry.group.executable.dependencies] -pyinstaller = ">=5.11.0,<5.12.0" -dunamai = ">=1.16.1,<1.17.0" +pyinstaller = {version=">=6.0.0,<7.0.0", python=">=3.9,<3.15"} +dunamai = ">=1.18.0,<1.27.0" + +[tool.poetry.group.dev.dependencies] +ruff = "0.11.7" [tool.pytest.ini_options] log_cli = true @@ -59,9 +74,77 @@ enable = true strict = true bump = true metadata = false +fix-shallow-repository=true vcs = "git" style = "pep440" +[tool.ruff] +line-length = 120 +target-version = "py39" + +[tool.ruff.lint] +extend-select = [ + "E", # pycodestyle errors + "W", # pycodestyle warnings + "F", # Pyflakes + "I", # isort + "N", # pep8 naming + "C90", # flake8-comprehensions + "B", # flake8-bugbear + "Q", # flake8-quotes + "S", # flake8-bandit + "ASYNC", # flake8-async + "ANN", # flake8-annotations + "C", + "BLE", + "ERA", + "ICN", + "INP", + "ISC", + "NPY", + "PGH", + "PIE", + "RET", + "RSE", + "RUF", + "SIM", + "T10", + "T20", + "TID", + "YTT", + "LOG", + "G", + "UP", + "DTZ", + "PYI", + "PT", + "SLOT", + "TC", +] +ignore = [ + "ANN002", # Missing type annotation for `*args` + "ANN003", # Missing type annotation for `**kwargs` + "ANN401", # Dynamically typed expressions (typing.Any) + "ISC001", # Conflicts with ruff format + "S105", # False positives + "PT012", # `pytest.raises()` block should contain a single simple statement +] + +[tool.ruff.lint.flake8-quotes] +docstring-quotes = "double" +multiline-quotes = "double" +inline-quotes = "single" + +[tool.ruff.lint.flake8-tidy-imports] +ban-relative-imports = "all" + +[tool.ruff.lint.per-file-ignores] +"tests/*.py" = ["S101", "S105"] +"cycode/*.py" = ["BLE001"] + +[tool.ruff.format] +quote-style = "single" + [build-system] -requires = ["poetry-core>=1.0.0", "poetry-dynamic-versioning"] +requires = ["poetry-core>=2.0.0", "poetry-dynamic-versioning>=1.0.0,<2.0.0"] build-backend = "poetry_dynamic_versioning.backend" diff --git a/tests/__init__.py b/tests/__init__.py index b475f6f5..617049bb 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -16,19 +16,15 @@ POD_MOCK = { 'metadata': { - "name": "pod-template-123xyz", - "namespace": "default", + 'name': 'pod-template-123xyz', + 'namespace': 'default', 'ownerReferences': [ { - "kind": "Deployment", - "name": "nginx-deployment", + 'kind': 'Deployment', + 'name': 'nginx-deployment', } - ] + ], } } K8S_POD_MOCK = K8SResource('pod-template-123xyz', 'pod', 'default', POD_MOCK) - - -def list_to_str(values): - return ",".join([str(val) for val in values]) diff --git a/tests/cli/apps/__init__.py b/tests/cli/apps/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cli/apps/mcp/__init__.py b/tests/cli/apps/mcp/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cli/apps/mcp/test_mcp_command.py b/tests/cli/apps/mcp/test_mcp_command.py new file mode 100644 index 00000000..ebcc2373 --- /dev/null +++ b/tests/cli/apps/mcp/test_mcp_command.py @@ -0,0 +1,315 @@ +import json +import os +import sys +from unittest.mock import AsyncMock, patch + +import pytest + +if sys.version_info < (3, 10): + pytest.skip('MCP requires Python 3.10+', allow_module_level=True) + +from cycode.cli.apps.mcp.mcp_command import ( + _sanitize_file_path, + _TempFilesManager, +) + +pytestmark = pytest.mark.anyio + + +@pytest.fixture +def anyio_backend() -> str: + return 'asyncio' + + +# --- _sanitize_file_path input validation --- + + +def test_sanitize_file_path_rejects_empty_string() -> None: + with pytest.raises(ValueError, match='non-empty string'): + _sanitize_file_path('') + + +def test_sanitize_file_path_rejects_none() -> None: + with pytest.raises(ValueError, match='non-empty string'): + _sanitize_file_path(None) + + +def test_sanitize_file_path_rejects_non_string() -> None: + with pytest.raises(ValueError, match='non-empty string'): + _sanitize_file_path(123) + + +def test_sanitize_file_path_strips_null_bytes() -> None: + result = _sanitize_file_path('foo/bar\x00baz.py') + assert '\x00' not in result + + +def test_sanitize_file_path_passes_valid_path_through() -> None: + result = _sanitize_file_path('src/main.py') + assert os.path.normpath(result) == os.path.normpath('src/main.py') + + +# --- _TempFilesManager: path traversal prevention --- +# +# _sanitize_file_path delegates to pathvalidate which does NOT block +# path traversal (../ passes through). The real security boundary is +# the normpath containment check in _TempFilesManager.__enter__ (lines 136-139). +# These tests verify that the two layers together prevent escaping the temp dir. + + +def test_traversal_simple_dotdot_rejected() -> None: + """../../../etc/passwd must not escape the temp directory.""" + files = { + '../../../etc/passwd': 'malicious', + 'safe.py': 'ok', + } + with _TempFilesManager(files, 'test-traversal') as temp_files: + assert len(temp_files) == 1 + assert temp_files[0].endswith('safe.py') + for tf in temp_files: + assert '/etc/passwd' not in tf + + +def test_traversal_backslash_dotdot_rejected() -> None: + """..\\..\\windows\\system32 must not escape the temp directory.""" + files = { + '..\\..\\windows\\system32\\config': 'malicious', + 'safe.py': 'ok', + } + with _TempFilesManager(files, 'test-backslash') as temp_files: + assert len(temp_files) == 1 + assert temp_files[0].endswith('safe.py') + + +def test_traversal_embedded_dotdot_rejected() -> None: + """foo/../../../etc/passwd resolves outside temp dir and must be rejected.""" + files = { + 'foo/../../../etc/passwd': 'malicious', + 'safe.py': 'ok', + } + with _TempFilesManager(files, 'test-embedded') as temp_files: + assert len(temp_files) == 1 + assert temp_files[0].endswith('safe.py') + + +def test_traversal_absolute_path_rejected() -> None: + """Absolute paths must not be written outside the temp directory.""" + files = { + '/etc/passwd': 'malicious', + 'safe.py': 'ok', + } + with _TempFilesManager(files, 'test-absolute') as temp_files: + assert len(temp_files) == 1 + assert temp_files[0].endswith('safe.py') + + +def test_traversal_dotdot_only_rejected() -> None: + """A bare '..' path must be rejected.""" + files = { + '..': 'malicious', + 'safe.py': 'ok', + } + with _TempFilesManager(files, 'test-bare-dotdot') as temp_files: + assert len(temp_files) == 1 + + +def test_traversal_all_malicious_raises() -> None: + """If every file path is a traversal attempt, no files are created and ValueError is raised.""" + files = { + '../../../etc/passwd': 'malicious', + '../../shadow': 'also malicious', + } + with pytest.raises(ValueError, match='No valid files'), _TempFilesManager(files, 'test-all-malicious'): + pass + + +def test_all_created_files_are_inside_temp_dir() -> None: + """Every created file must be under the temp base directory.""" + files = { + 'a.py': 'aaa', + 'sub/b.py': 'bbb', + 'sub/deep/c.py': 'ccc', + } + manager = _TempFilesManager(files, 'test-containment') + with manager as temp_files: + base = os.path.normcase(os.path.normpath(manager.temp_base_dir)) + for tf in temp_files: + normalized = os.path.normcase(os.path.normpath(tf)) + assert normalized.startswith(base + os.sep), f'{tf} escaped temp dir {base}' + + +def test_mixed_valid_and_traversal_only_creates_valid() -> None: + """Valid files are created, traversal attempts are silently skipped.""" + files = { + '../escape.py': 'bad', + 'legit.py': 'good', + 'foo/../../escape2.py': 'bad', + 'src/app.py': 'good', + } + manager = _TempFilesManager(files, 'test-mixed') + with manager as temp_files: + base = os.path.normcase(os.path.normpath(manager.temp_base_dir)) + assert len(temp_files) == 2 + for tf in temp_files: + assert os.path.normcase(os.path.normpath(tf)).startswith(base + os.sep) + basenames = [os.path.basename(tf) for tf in temp_files] + assert 'legit.py' in basenames + assert 'app.py' in basenames + + +# --- _TempFilesManager: general functionality --- + + +def test_temp_files_manager_creates_files() -> None: + files = { + 'test1.py': 'print("hello")', + 'subdir/test2.js': 'console.log("world")', + } + with _TempFilesManager(files, 'test-call-id') as temp_files: + assert len(temp_files) == 2 + for tf in temp_files: + assert os.path.exists(tf) + + +def test_temp_files_manager_writes_correct_content() -> None: + files = {'hello.py': 'print("hello world")'} + with _TempFilesManager(files, 'test-content') as temp_files, open(temp_files[0]) as f: + assert f.read() == 'print("hello world")' + + +def test_temp_files_manager_cleans_up_on_exit() -> None: + files = {'cleanup.py': 'code'} + manager = _TempFilesManager(files, 'test-cleanup') + with manager as temp_files: + temp_dir = manager.temp_base_dir + assert os.path.exists(temp_dir) + assert len(temp_files) == 1 + assert not os.path.exists(temp_dir) + + +def test_temp_files_manager_empty_path_raises() -> None: + files = {'': 'empty path'} + with pytest.raises(ValueError, match='No valid files'), _TempFilesManager(files, 'test-empty-path'): + pass + + +def test_temp_files_manager_preserves_subdirectory_structure() -> None: + files = { + 'src/main.py': 'main', + 'src/utils/helper.py': 'helper', + } + with _TempFilesManager(files, 'test-dirs') as temp_files: + assert len(temp_files) == 2 + paths = [os.path.basename(tf) for tf in temp_files] + assert 'main.py' in paths + assert 'helper.py' in paths + + +# --- _run_cycode_command (async) --- + + +@pytest.mark.anyio +async def test_run_cycode_command_returns_dict() -> None: + from cycode.cli.apps.mcp.mcp_command import _run_cycode_command + + mock_process = AsyncMock() + mock_process.communicate.return_value = (b'', b'error output') + mock_process.returncode = 1 + + with patch('asyncio.create_subprocess_exec', return_value=mock_process): + result = await _run_cycode_command('--invalid-flag-for-test') + assert isinstance(result, dict) + assert 'error' in result + + +@pytest.mark.anyio +async def test_run_cycode_command_parses_json_output() -> None: + from cycode.cli.apps.mcp.mcp_command import _run_cycode_command + + mock_process = AsyncMock() + mock_process.communicate.return_value = (b'{"status": "ok"}', b'') + mock_process.returncode = 0 + + with patch('asyncio.create_subprocess_exec', return_value=mock_process): + result = await _run_cycode_command('version') + assert result == {'status': 'ok'} + + +@pytest.mark.anyio +async def test_run_cycode_command_handles_invalid_json() -> None: + from cycode.cli.apps.mcp.mcp_command import _run_cycode_command + + mock_process = AsyncMock() + mock_process.communicate.return_value = (b'not json{', b'') + mock_process.returncode = 0 + + with patch('asyncio.create_subprocess_exec', return_value=mock_process): + result = await _run_cycode_command('version') + assert result['error'] == 'Failed to parse JSON output' + + +@pytest.mark.anyio +async def test_run_cycode_command_timeout() -> None: + import asyncio + + from cycode.cli.apps.mcp.mcp_command import _run_cycode_command + + async def slow_communicate() -> tuple[bytes, bytes]: + await asyncio.sleep(10) + return b'', b'' + + mock_process = AsyncMock() + mock_process.communicate = slow_communicate + + with patch('asyncio.create_subprocess_exec', return_value=mock_process): + result = await _run_cycode_command('status', timeout=0.001) + assert isinstance(result, dict) + assert 'error' in result + assert 'timeout' in result['error'].lower() + + +# --- _cycode_scan_tool --- + + +@pytest.mark.anyio +async def test_cycode_scan_tool_no_files() -> None: + from cycode.cli.apps.mcp.mcp_command import _cycode_scan_tool + from cycode.cli.cli_types import ScanTypeOption + + result = await _cycode_scan_tool(ScanTypeOption.SECRET, {}) + parsed = json.loads(result) + assert 'error' in parsed + assert 'No files provided' in parsed['error'] + + +@pytest.mark.anyio +async def test_cycode_scan_tool_invalid_files() -> None: + from cycode.cli.apps.mcp.mcp_command import _cycode_scan_tool + from cycode.cli.cli_types import ScanTypeOption + + result = await _cycode_scan_tool(ScanTypeOption.SECRET, {'': 'content'}) + parsed = json.loads(result) + assert 'error' in parsed + + +# --- _create_mcp_server --- + + +def test_create_mcp_server() -> None: + from cycode.cli.apps.mcp.mcp_command import _create_mcp_server + + server = _create_mcp_server('127.0.0.1', 8000) + assert server is not None + assert server.name == 'cycode' + + +def test_create_mcp_server_registers_tools() -> None: + from cycode.cli.apps.mcp.mcp_command import _create_mcp_server + + server = _create_mcp_server('127.0.0.1', 8000) + tool_names = [t.name for t in server._tool_manager._tools.values()] + assert 'cycode_status' in tool_names + assert 'cycode_secret_scan' in tool_names + assert 'cycode_sca_scan' in tool_names + assert 'cycode_iac_scan' in tool_names + assert 'cycode_sast_scan' in tool_names diff --git a/tests/cli/commands/__init__.py b/tests/cli/commands/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cli/commands/ai_guardrails/__init__.py b/tests/cli/commands/ai_guardrails/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cli/commands/ai_guardrails/scan/__init__.py b/tests/cli/commands/ai_guardrails/scan/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cli/commands/ai_guardrails/scan/test_handlers.py b/tests/cli/commands/ai_guardrails/scan/test_handlers.py new file mode 100644 index 00000000..1adfe25b --- /dev/null +++ b/tests/cli/commands/ai_guardrails/scan/test_handlers.py @@ -0,0 +1,364 @@ +"""Tests for AI guardrails handlers.""" + +from typing import Any +from unittest.mock import MagicMock, patch + +import pytest +import typer + +from cycode.cli.apps.ai_guardrails.scan.handlers import ( + handle_before_mcp_execution, + handle_before_read_file, + handle_before_submit_prompt, +) +from cycode.cli.apps.ai_guardrails.scan.payload import AIHookPayload +from cycode.cli.apps.ai_guardrails.scan.types import AIHookOutcome, BlockReason + + +@pytest.fixture +def mock_ctx() -> MagicMock: + """Create a mock Typer context.""" + ctx = MagicMock(spec=typer.Context) + ctx.obj = { + 'ai_security_client': MagicMock(), + 'scan_type': 'secret', + } + return ctx + + +@pytest.fixture +def mock_payload() -> AIHookPayload: + """Create a mock AIHookPayload.""" + return AIHookPayload( + event_name='prompt', + conversation_id='test-conv-id', + generation_id='test-gen-id', + ide_user_email='test@example.com', + model='gpt-4', + ide_provider='cursor', + ide_version='1.0.0', + prompt='Test prompt', + ) + + +@pytest.fixture +def default_policy() -> dict[str, Any]: + """Create a default policy dict.""" + return { + 'mode': 'block', + 'fail_open': True, + 'secrets': {'max_bytes': 200000}, + 'prompt': {'enabled': True, 'action': 'block'}, + 'file_read': {'enabled': True, 'action': 'block', 'scan_content': True, 'deny_globs': []}, + 'mcp': {'enabled': True, 'action': 'block', 'scan_arguments': True}, + } + + +# Tests for handle_before_submit_prompt + + +def test_handle_before_submit_prompt_disabled( + mock_ctx: MagicMock, mock_payload: AIHookPayload, default_policy: dict[str, Any] +) -> None: + """Test that disabled prompt scanning allows the prompt.""" + default_policy['prompt']['enabled'] = False + + result = handle_before_submit_prompt(mock_ctx, mock_payload, default_policy) + + assert result == {'continue': True} + mock_ctx.obj['ai_security_client'].create_event.assert_called_once() + + +@patch('cycode.cli.apps.ai_guardrails.scan.handlers._scan_text_for_secrets') +def test_handle_before_submit_prompt_no_secrets( + mock_scan: MagicMock, mock_ctx: MagicMock, mock_payload: AIHookPayload, default_policy: dict[str, Any] +) -> None: + """Test that prompt with no secrets is allowed.""" + mock_scan.return_value = (None, 'scan-id-123') + + result = handle_before_submit_prompt(mock_ctx, mock_payload, default_policy) + + assert result == {'continue': True} + mock_ctx.obj['ai_security_client'].create_event.assert_called_once() + call_args = mock_ctx.obj['ai_security_client'].create_event.call_args + # outcome is arg[2], scan_id and block_reason are kwargs + assert call_args.args[2] == AIHookOutcome.ALLOWED + assert call_args.kwargs['scan_id'] == 'scan-id-123' + assert call_args.kwargs['block_reason'] is None + + +@patch('cycode.cli.apps.ai_guardrails.scan.handlers._scan_text_for_secrets') +def test_handle_before_submit_prompt_with_secrets_blocked( + mock_scan: MagicMock, mock_ctx: MagicMock, mock_payload: AIHookPayload, default_policy: dict[str, Any] +) -> None: + """Test that prompt with secrets is blocked.""" + mock_scan.return_value = ('Found 1 secret: API key', 'scan-id-456') + + result = handle_before_submit_prompt(mock_ctx, mock_payload, default_policy) + + assert result['continue'] is False + assert 'Found 1 secret: API key' in result['user_message'] + mock_ctx.obj['ai_security_client'].create_event.assert_called_once() + call_args = mock_ctx.obj['ai_security_client'].create_event.call_args + assert call_args.args[2] == AIHookOutcome.BLOCKED + assert call_args.kwargs['block_reason'] == BlockReason.SECRETS_IN_PROMPT + + +@patch('cycode.cli.apps.ai_guardrails.scan.handlers._scan_text_for_secrets') +def test_handle_before_submit_prompt_with_secrets_warned( + mock_scan: MagicMock, mock_ctx: MagicMock, mock_payload: AIHookPayload, default_policy: dict[str, Any] +) -> None: + """Test that prompt with secrets in warn mode is allowed.""" + default_policy['prompt']['action'] = 'warn' + mock_scan.return_value = ('Found 1 secret: API key', 'scan-id-789') + + result = handle_before_submit_prompt(mock_ctx, mock_payload, default_policy) + + assert result == {'continue': True} + mock_ctx.obj['ai_security_client'].create_event.assert_called_once() + call_args = mock_ctx.obj['ai_security_client'].create_event.call_args + assert call_args.args[2] == AIHookOutcome.WARNED + + +@patch('cycode.cli.apps.ai_guardrails.scan.handlers._scan_text_for_secrets') +def test_handle_before_submit_prompt_scan_failure_fail_open( + mock_scan: MagicMock, mock_ctx: MagicMock, mock_payload: AIHookPayload, default_policy: dict[str, Any] +) -> None: + """Test that scan failure with fail_open=True allows the prompt.""" + mock_scan.side_effect = RuntimeError('Scan failed') + default_policy['fail_open'] = True + + with pytest.raises(RuntimeError): + handle_before_submit_prompt(mock_ctx, mock_payload, default_policy) + + # Event should be tracked even on exception + mock_ctx.obj['ai_security_client'].create_event.assert_called_once() + call_args = mock_ctx.obj['ai_security_client'].create_event.call_args + assert call_args.args[2] == AIHookOutcome.ALLOWED + # block_reason is set for tracking even when fail_open allows the action + assert call_args.kwargs['block_reason'] == BlockReason.SCAN_FAILURE + + +@patch('cycode.cli.apps.ai_guardrails.scan.handlers._scan_text_for_secrets') +def test_handle_before_submit_prompt_scan_failure_fail_closed( + mock_scan: MagicMock, mock_ctx: MagicMock, mock_payload: AIHookPayload, default_policy: dict[str, Any] +) -> None: + """Test that scan failure with fail_open=False blocks the prompt.""" + mock_scan.side_effect = RuntimeError('Scan failed') + default_policy['fail_open'] = False + + with pytest.raises(RuntimeError): + handle_before_submit_prompt(mock_ctx, mock_payload, default_policy) + + # Event should be tracked even on exception + mock_ctx.obj['ai_security_client'].create_event.assert_called_once() + call_args = mock_ctx.obj['ai_security_client'].create_event.call_args + assert call_args.args[2] == AIHookOutcome.BLOCKED + assert call_args.kwargs['block_reason'] == BlockReason.SCAN_FAILURE + + +# Tests for handle_before_read_file + + +def test_handle_before_read_file_disabled(mock_ctx: MagicMock, default_policy: dict[str, Any]) -> None: + """Test that disabled file read scanning allows the file.""" + default_policy['file_read']['enabled'] = False + payload = AIHookPayload( + event_name='file_read', + ide_provider='cursor', + file_path='/path/to/file.txt', + ) + + result = handle_before_read_file(mock_ctx, payload, default_policy) + + assert result == {'permission': 'allow'} + + +@patch('cycode.cli.apps.ai_guardrails.scan.handlers.is_denied_path') +def test_handle_before_read_file_sensitive_path( + mock_is_denied: MagicMock, mock_ctx: MagicMock, default_policy: dict[str, Any] +) -> None: + """Test that sensitive path is blocked.""" + mock_is_denied.return_value = True + payload = AIHookPayload( + event_name='file_read', + ide_provider='cursor', + file_path='/path/to/.env', + ) + + result = handle_before_read_file(mock_ctx, payload, default_policy) + + assert result['permission'] == 'deny' + assert '.env' in result['user_message'] + mock_ctx.obj['ai_security_client'].create_event.assert_called_once() + call_args = mock_ctx.obj['ai_security_client'].create_event.call_args + assert call_args.args[2] == AIHookOutcome.BLOCKED + assert call_args.kwargs['block_reason'] == BlockReason.SENSITIVE_PATH + assert call_args.kwargs['file_path'] == '/path/to/.env' + + +@patch('cycode.cli.apps.ai_guardrails.scan.handlers.is_denied_path') +@patch('cycode.cli.apps.ai_guardrails.scan.handlers._scan_path_for_secrets') +def test_handle_before_read_file_no_secrets( + mock_scan: MagicMock, mock_is_denied: MagicMock, mock_ctx: MagicMock, default_policy: dict[str, Any] +) -> None: + """Test that file with no secrets is allowed.""" + mock_is_denied.return_value = False + mock_scan.return_value = (None, 'scan-id-123') + payload = AIHookPayload( + event_name='file_read', + ide_provider='cursor', + file_path='/path/to/file.txt', + ) + + result = handle_before_read_file(mock_ctx, payload, default_policy) + + assert result == {'permission': 'allow'} + call_args = mock_ctx.obj['ai_security_client'].create_event.call_args + assert call_args.args[2] == AIHookOutcome.ALLOWED + assert call_args.kwargs['file_path'] == '/path/to/file.txt' + + +@patch('cycode.cli.apps.ai_guardrails.scan.handlers.is_denied_path') +@patch('cycode.cli.apps.ai_guardrails.scan.handlers._scan_path_for_secrets') +def test_handle_before_read_file_with_secrets( + mock_scan: MagicMock, mock_is_denied: MagicMock, mock_ctx: MagicMock, default_policy: dict[str, Any] +) -> None: + """Test that file with secrets is blocked.""" + mock_is_denied.return_value = False + mock_scan.return_value = ('Found 1 secret: password', 'scan-id-456') + payload = AIHookPayload( + event_name='file_read', + ide_provider='cursor', + file_path='/path/to/file.txt', + ) + + result = handle_before_read_file(mock_ctx, payload, default_policy) + + assert result['permission'] == 'deny' + assert 'Found 1 secret: password' in result['user_message'] + call_args = mock_ctx.obj['ai_security_client'].create_event.call_args + assert call_args.args[2] == AIHookOutcome.BLOCKED + assert call_args.kwargs['block_reason'] == BlockReason.SECRETS_IN_FILE + assert call_args.kwargs['file_path'] == '/path/to/file.txt' + + +@patch('cycode.cli.apps.ai_guardrails.scan.handlers.is_denied_path') +@patch('cycode.cli.apps.ai_guardrails.scan.handlers._scan_path_for_secrets') +def test_handle_before_read_file_scan_disabled( + mock_scan: MagicMock, mock_is_denied: MagicMock, mock_ctx: MagicMock, default_policy: dict[str, Any] +) -> None: + """Test that file is allowed when content scanning is disabled.""" + mock_is_denied.return_value = False + default_policy['file_read']['scan_content'] = False + payload = AIHookPayload( + event_name='file_read', + ide_provider='cursor', + file_path='/path/to/file.txt', + ) + + result = handle_before_read_file(mock_ctx, payload, default_policy) + + assert result == {'permission': 'allow'} + mock_scan.assert_not_called() + + +# Tests for handle_before_mcp_execution + + +def test_handle_before_mcp_execution_disabled(mock_ctx: MagicMock, default_policy: dict[str, Any]) -> None: + """Test that disabled MCP scanning allows the execution.""" + default_policy['mcp']['enabled'] = False + payload = AIHookPayload( + event_name='mcp_execution', + ide_provider='cursor', + mcp_tool_name='test_tool', + mcp_arguments={'arg1': 'value1'}, + ) + + result = handle_before_mcp_execution(mock_ctx, payload, default_policy) + + assert result == {'permission': 'allow'} + + +@patch('cycode.cli.apps.ai_guardrails.scan.handlers._scan_text_for_secrets') +def test_handle_before_mcp_execution_no_secrets( + mock_scan: MagicMock, mock_ctx: MagicMock, default_policy: dict[str, Any] +) -> None: + """Test that MCP execution with no secrets is allowed.""" + mock_scan.return_value = (None, 'scan-id-123') + payload = AIHookPayload( + event_name='mcp_execution', + ide_provider='cursor', + mcp_tool_name='test_tool', + mcp_arguments={'arg1': 'value1'}, + ) + + result = handle_before_mcp_execution(mock_ctx, payload, default_policy) + + assert result == {'permission': 'allow'} + call_args = mock_ctx.obj['ai_security_client'].create_event.call_args + assert call_args.args[2] == AIHookOutcome.ALLOWED + + +@patch('cycode.cli.apps.ai_guardrails.scan.handlers._scan_text_for_secrets') +def test_handle_before_mcp_execution_with_secrets_blocked( + mock_scan: MagicMock, mock_ctx: MagicMock, default_policy: dict[str, Any] +) -> None: + """Test that MCP execution with secrets is blocked.""" + mock_scan.return_value = ('Found 1 secret: token', 'scan-id-456') + payload = AIHookPayload( + event_name='mcp_execution', + ide_provider='cursor', + mcp_tool_name='test_tool', + mcp_arguments={'arg1': 'secret_token_12345'}, + ) + + result = handle_before_mcp_execution(mock_ctx, payload, default_policy) + + assert result['permission'] == 'deny' + assert 'Found 1 secret: token' in result['user_message'] + call_args = mock_ctx.obj['ai_security_client'].create_event.call_args + assert call_args.args[2] == AIHookOutcome.BLOCKED + assert call_args.kwargs['block_reason'] == BlockReason.SECRETS_IN_MCP_ARGS + + +@patch('cycode.cli.apps.ai_guardrails.scan.handlers._scan_text_for_secrets') +def test_handle_before_mcp_execution_with_secrets_warned( + mock_scan: MagicMock, mock_ctx: MagicMock, default_policy: dict[str, Any] +) -> None: + """Test that MCP execution with secrets in warn mode asks permission.""" + mock_scan.return_value = ('Found 1 secret: token', 'scan-id-789') + default_policy['mcp']['action'] = 'warn' + payload = AIHookPayload( + event_name='mcp_execution', + ide_provider='cursor', + mcp_tool_name='test_tool', + mcp_arguments={'arg1': 'secret_token_12345'}, + ) + + result = handle_before_mcp_execution(mock_ctx, payload, default_policy) + + assert result['permission'] == 'ask' + assert 'Found 1 secret: token' in result['user_message'] + call_args = mock_ctx.obj['ai_security_client'].create_event.call_args + assert call_args.args[2] == AIHookOutcome.WARNED + + +@patch('cycode.cli.apps.ai_guardrails.scan.handlers._scan_text_for_secrets') +def test_handle_before_mcp_execution_scan_disabled( + mock_scan: MagicMock, mock_ctx: MagicMock, default_policy: dict[str, Any] +) -> None: + """Test that MCP execution is allowed when argument scanning is disabled.""" + default_policy['mcp']['scan_arguments'] = False + payload = AIHookPayload( + event_name='mcp_execution', + ide_provider='cursor', + mcp_tool_name='test_tool', + mcp_arguments={'arg1': 'value1'}, + ) + + result = handle_before_mcp_execution(mock_ctx, payload, default_policy) + + assert result == {'permission': 'allow'} + mock_scan.assert_not_called() diff --git a/tests/cli/commands/ai_guardrails/scan/test_payload.py b/tests/cli/commands/ai_guardrails/scan/test_payload.py new file mode 100644 index 00000000..27c3010f --- /dev/null +++ b/tests/cli/commands/ai_guardrails/scan/test_payload.py @@ -0,0 +1,376 @@ +"""Tests for AI hook payload normalization.""" + +import pytest +from pytest_mock import MockerFixture + +from cycode.cli.apps.ai_guardrails.scan.payload import AIHookPayload +from cycode.cli.apps.ai_guardrails.scan.types import AiHookEventType + + +def test_from_cursor_payload_prompt_event() -> None: + """Test conversion of Cursor beforeSubmitPrompt payload.""" + cursor_payload = { + 'hook_event_name': 'beforeSubmitPrompt', + 'conversation_id': 'conv-123', + 'generation_id': 'gen-456', + 'user_email': 'user@example.com', + 'model': 'gpt-4', + 'cursor_version': '0.42.0', + 'prompt': 'Test prompt', + } + + unified = AIHookPayload.from_cursor_payload(cursor_payload) + + assert unified.event_name == AiHookEventType.PROMPT + assert unified.conversation_id == 'conv-123' + assert unified.generation_id == 'gen-456' + assert unified.ide_user_email == 'user@example.com' + assert unified.model == 'gpt-4' + assert unified.ide_provider == 'cursor' + assert unified.ide_version == '0.42.0' + assert unified.prompt == 'Test prompt' + + +def test_from_cursor_payload_file_read_event() -> None: + """Test conversion of Cursor beforeReadFile payload.""" + cursor_payload = { + 'hook_event_name': 'beforeReadFile', + 'conversation_id': 'conv-123', + 'file_path': '/path/to/secret.env', + } + + unified = AIHookPayload.from_cursor_payload(cursor_payload) + + assert unified.event_name == AiHookEventType.FILE_READ + assert unified.file_path == '/path/to/secret.env' + assert unified.ide_provider == 'cursor' + + +def test_from_cursor_payload_mcp_execution_event() -> None: + """Test conversion of Cursor beforeMCPExecution payload.""" + cursor_payload = { + 'hook_event_name': 'beforeMCPExecution', + 'conversation_id': 'conv-123', + 'command': 'GitLab', + 'tool_name': 'discussion_list', + 'arguments': {'resource_type': 'merge_request', 'parent_id': 'organization/repo', 'resource_id': '4'}, + } + + unified = AIHookPayload.from_cursor_payload(cursor_payload) + + assert unified.event_name == AiHookEventType.MCP_EXECUTION + assert unified.mcp_server_name == 'GitLab' + assert unified.mcp_tool_name == 'discussion_list' + assert unified.mcp_arguments == { + 'resource_type': 'merge_request', + 'parent_id': 'organization/repo', + 'resource_id': '4', + } + + +def test_from_cursor_payload_with_alternative_field_names() -> None: + """Test that alternative field names are handled (path vs file_path, etc.).""" + cursor_payload = { + 'hook_event_name': 'beforeReadFile', + 'path': '/alternative/path.txt', # Alternative to file_path + } + + unified = AIHookPayload.from_cursor_payload(cursor_payload) + assert unified.file_path == '/alternative/path.txt' + + cursor_payload = { + 'hook_event_name': 'beforeMCPExecution', + 'tool': 'my_tool', # Alternative to tool_name + 'tool_input': {'key': 'value'}, # Alternative to arguments + } + + unified = AIHookPayload.from_cursor_payload(cursor_payload) + assert unified.mcp_tool_name == 'my_tool' + assert unified.mcp_arguments == {'key': 'value'} + + +def test_from_cursor_payload_unknown_event() -> None: + """Test that unknown event names are passed through as-is.""" + cursor_payload = { + 'hook_event_name': 'unknownEvent', + 'conversation_id': 'conv-123', + } + + unified = AIHookPayload.from_cursor_payload(cursor_payload) + # Unknown events fall back to original name + assert unified.event_name == 'unknownEvent' + + +def test_from_payload_cursor() -> None: + """Test from_payload dispatcher with Cursor tool.""" + cursor_payload = { + 'hook_event_name': 'beforeSubmitPrompt', + 'prompt': 'test', + } + + unified = AIHookPayload.from_payload(cursor_payload, tool='cursor') + assert unified.event_name == AiHookEventType.PROMPT + assert unified.ide_provider == 'cursor' + + +def test_from_payload_unsupported_tool() -> None: + """Test from_payload raises ValueError for unsupported tools.""" + payload = {'hook_event_name': 'someEvent'} + + with pytest.raises(ValueError, match='Unsupported IDE/tool: unsupported'): + AIHookPayload.from_payload(payload, tool='unsupported') + + +def test_from_cursor_payload_empty_fields() -> None: + """Test handling of empty/missing fields.""" + cursor_payload = { + 'hook_event_name': 'beforeSubmitPrompt', + # Most fields missing + } + + unified = AIHookPayload.from_cursor_payload(cursor_payload) + + assert unified.event_name == AiHookEventType.PROMPT + assert unified.conversation_id is None + assert unified.prompt == '' # Default to empty string + assert unified.ide_provider == 'cursor' + + +# Claude Code payload tests + + +def test_from_claude_code_payload_prompt_event() -> None: + """Test conversion of Claude Code UserPromptSubmit payload.""" + claude_payload = { + 'hook_event_name': 'UserPromptSubmit', + 'session_id': 'session-123', + 'prompt': 'Test prompt for Claude Code', + } + + unified = AIHookPayload.from_claude_code_payload(claude_payload) + + assert unified.event_name == AiHookEventType.PROMPT + assert unified.conversation_id == 'session-123' + assert unified.ide_provider == 'claude-code' + assert unified.prompt == 'Test prompt for Claude Code' + + +def test_from_claude_code_payload_file_read_event() -> None: + """Test conversion of Claude Code PreToolUse with Read tool.""" + claude_payload = { + 'hook_event_name': 'PreToolUse', + 'session_id': 'session-456', + 'tool_name': 'Read', + 'tool_input': {'file_path': '/path/to/secret.env'}, + } + + unified = AIHookPayload.from_claude_code_payload(claude_payload) + + assert unified.event_name == AiHookEventType.FILE_READ + assert unified.file_path == '/path/to/secret.env' + assert unified.ide_provider == 'claude-code' + assert unified.mcp_tool_name is None + + +def test_from_claude_code_payload_mcp_execution_event() -> None: + """Test conversion of Claude Code PreToolUse with MCP tool.""" + claude_payload = { + 'hook_event_name': 'PreToolUse', + 'session_id': 'session-789', + 'tool_name': 'mcp__gitlab__discussion_list', + 'tool_input': {'resource_type': 'merge_request', 'parent_id': 'org/repo', 'resource_id': '4'}, + } + + unified = AIHookPayload.from_payload(claude_payload, tool='claude-code') + + assert unified.event_name == AiHookEventType.MCP_EXECUTION + assert unified.mcp_server_name == 'gitlab' + assert unified.mcp_tool_name == 'discussion_list' + assert unified.mcp_arguments == {'resource_type': 'merge_request', 'parent_id': 'org/repo', 'resource_id': '4'} + assert unified.ide_provider == 'claude-code' + + +def test_from_claude_code_payload_empty_fields() -> None: + """Test handling of empty/missing fields for Claude Code.""" + claude_payload = { + 'hook_event_name': 'UserPromptSubmit', + # Most fields missing + } + + unified = AIHookPayload.from_claude_code_payload(claude_payload) + + assert unified.event_name == AiHookEventType.PROMPT + assert unified.conversation_id is None + assert unified.prompt == '' # Default to empty string + assert unified.ide_provider == 'claude-code' + + +# Claude Code transcript extraction tests + + +def test_from_claude_code_payload_extracts_from_transcript(mocker: MockerFixture) -> None: + """Test that version, model, and generation_id are extracted from transcript file.""" + transcript_content = ( + b'{"type":"user","version":"2.1.20","uuid":"user-uuid-1","message":{"role":"user","content":"hello"}}\n' + b'{"type":"assistant","message":{"model":"claude-opus-4-5-20251101","role":"assistant",' + b'"content":[{"type":"text","text":"Hi!"}]},"uuid":"assistant-uuid-1"}\n' + b'{"type":"user","version":"2.1.20","uuid":"user-uuid-2","message":{"role":"user","content":"test prompt"}}\n' + ) + mock_path = mocker.patch('cycode.cli.apps.ai_guardrails.scan.payload.Path') + mock_path.return_value.exists.return_value = True + mock_path.return_value.open.return_value.__enter__.return_value.seek = mocker.Mock() + mock_path.return_value.open.return_value.__enter__.return_value.tell.return_value = len(transcript_content) + mock_path.return_value.open.return_value.__enter__.return_value.read.return_value = transcript_content + + claude_payload = { + 'hook_event_name': 'UserPromptSubmit', + 'session_id': 'session-123', + 'prompt': 'test prompt', + 'transcript_path': '/mock/transcript.jsonl', + } + + unified = AIHookPayload.from_claude_code_payload(claude_payload) + + assert unified.ide_version == '2.1.20' + assert unified.model == 'claude-opus-4-5-20251101' + assert unified.generation_id == 'user-uuid-2' + + +def test_from_claude_code_payload_handles_missing_transcript(mocker: MockerFixture) -> None: + """Test that missing transcript file doesn't break payload parsing.""" + mock_path = mocker.patch('cycode.cli.apps.ai_guardrails.scan.payload.Path') + mock_path.return_value.exists.return_value = False + + claude_payload = { + 'hook_event_name': 'UserPromptSubmit', + 'session_id': 'session-123', + 'prompt': 'test', + 'transcript_path': '/nonexistent/path/transcript.jsonl', + } + + unified = AIHookPayload.from_claude_code_payload(claude_payload) + + assert unified.ide_version is None + assert unified.model is None + assert unified.generation_id is None + assert unified.conversation_id == 'session-123' + assert unified.prompt == 'test' + + +def test_from_claude_code_payload_handles_no_transcript_path() -> None: + """Test that absent transcript_path doesn't break payload parsing.""" + claude_payload = { + 'hook_event_name': 'UserPromptSubmit', + 'session_id': 'session-123', + 'prompt': 'test', + } + + unified = AIHookPayload.from_claude_code_payload(claude_payload) + + assert unified.ide_version is None + assert unified.model is None + assert unified.generation_id is None + + +def test_from_claude_code_payload_extracts_model_from_nested_message(mocker: MockerFixture) -> None: + """Test that model is extracted from nested message.model field.""" + transcript_content = ( + b'{"type":"assistant","message":{"model":"claude-sonnet-4-20250514",' + b'"role":"assistant","content":[]},"uuid":"uuid-1"}\n' + ) + + mock_path = mocker.patch('cycode.cli.apps.ai_guardrails.scan.payload.Path') + mock_path.return_value.exists.return_value = True + mock_path.return_value.open.return_value.__enter__.return_value.seek = mocker.Mock() + mock_path.return_value.open.return_value.__enter__.return_value.tell.return_value = len(transcript_content) + mock_path.return_value.open.return_value.__enter__.return_value.read.return_value = transcript_content + + claude_payload = { + 'hook_event_name': 'UserPromptSubmit', + 'prompt': 'test', + 'transcript_path': '/mock/transcript.jsonl', + } + + unified = AIHookPayload.from_claude_code_payload(claude_payload) + + assert unified.model == 'claude-sonnet-4-20250514' + + +def test_from_claude_code_payload_gets_latest_user_uuid(mocker: MockerFixture) -> None: + """Test that generation_id is the UUID of the latest user message.""" + transcript_content = b"""{"type":"user","uuid":"old-user-uuid","message":{"role":"user","content":"first"}} +{"type":"assistant","uuid":"assistant-uuid","message":{"role":"assistant","content":[]}} +{"type":"user","uuid":"latest-user-uuid","message":{"role":"user","content":"second"}} +{"type":"assistant","uuid":"last-assistant-uuid","message":{"role":"assistant","content":[]}} +""" + mock_path = mocker.patch('cycode.cli.apps.ai_guardrails.scan.payload.Path') + mock_path.return_value.exists.return_value = True + mock_path.return_value.open.return_value.__enter__.return_value.seek = mocker.Mock() + mock_path.return_value.open.return_value.__enter__.return_value.tell.return_value = len(transcript_content) + mock_path.return_value.open.return_value.__enter__.return_value.read.return_value = transcript_content + + claude_payload = { + 'hook_event_name': 'UserPromptSubmit', + 'prompt': 'test', + 'transcript_path': '/mock/transcript.jsonl', + } + + unified = AIHookPayload.from_claude_code_payload(claude_payload) + + assert unified.generation_id == 'latest-user-uuid' + + +# IDE detection tests + + +def test_is_payload_for_ide_claude_code_matches_claude_code() -> None: + """Test that Claude Code events match when expected IDE is claude-code.""" + payload = {'hook_event_name': 'UserPromptSubmit'} + assert AIHookPayload.is_payload_for_ide(payload, 'claude-code') is True + + payload = {'hook_event_name': 'PreToolUse'} + assert AIHookPayload.is_payload_for_ide(payload, 'claude-code') is True + + +def test_is_payload_for_ide_cursor_matches_cursor() -> None: + """Test that Cursor events match when expected IDE is cursor.""" + payload = {'hook_event_name': 'beforeSubmitPrompt'} + assert AIHookPayload.is_payload_for_ide(payload, 'cursor') is True + + payload = {'hook_event_name': 'beforeReadFile'} + assert AIHookPayload.is_payload_for_ide(payload, 'cursor') is True + + payload = {'hook_event_name': 'beforeMCPExecution'} + assert AIHookPayload.is_payload_for_ide(payload, 'cursor') is True + + +def test_is_payload_for_ide_claude_code_does_not_match_cursor() -> None: + """Test that Claude Code events don't match when expected IDE is cursor. + + This prevents double-processing when Cursor reads Claude Code hooks. + """ + payload = {'hook_event_name': 'UserPromptSubmit'} + assert AIHookPayload.is_payload_for_ide(payload, 'cursor') is False + + payload = {'hook_event_name': 'PreToolUse'} + assert AIHookPayload.is_payload_for_ide(payload, 'cursor') is False + + +def test_is_payload_for_ide_cursor_does_not_match_claude_code() -> None: + """Test that Cursor events don't match when expected IDE is claude-code.""" + payload = {'hook_event_name': 'beforeSubmitPrompt'} + assert AIHookPayload.is_payload_for_ide(payload, 'claude-code') is False + + payload = {'hook_event_name': 'beforeReadFile'} + assert AIHookPayload.is_payload_for_ide(payload, 'claude-code') is False + + +def test_is_payload_for_ide_empty_event_name() -> None: + """Test handling of empty or missing hook_event_name.""" + payload = {'hook_event_name': ''} + assert AIHookPayload.is_payload_for_ide(payload, 'cursor') is False + assert AIHookPayload.is_payload_for_ide(payload, 'claude-code') is False + + payload = {} + assert AIHookPayload.is_payload_for_ide(payload, 'cursor') is False + assert AIHookPayload.is_payload_for_ide(payload, 'claude-code') is False diff --git a/tests/cli/commands/ai_guardrails/scan/test_policy.py b/tests/cli/commands/ai_guardrails/scan/test_policy.py new file mode 100644 index 00000000..bbe884b0 --- /dev/null +++ b/tests/cli/commands/ai_guardrails/scan/test_policy.py @@ -0,0 +1,199 @@ +"""Tests for AI guardrails policy loading and management.""" + +from pathlib import Path +from typing import Optional +from unittest.mock import MagicMock, patch + +from pyfakefs.fake_filesystem import FakeFilesystem + +from cycode.cli.apps.ai_guardrails.scan.policy import ( + deep_merge, + get_policy_value, + load_defaults, + load_policy, + load_yaml_file, +) + + +def test_deep_merge_simple() -> None: + """Test deep merging two simple dictionaries.""" + base = {'a': 1, 'b': 2} + override = {'b': 3, 'c': 4} + result = deep_merge(base, override) + + assert result == {'a': 1, 'b': 3, 'c': 4} + + +def test_deep_merge_nested() -> None: + """Test deep merging nested dictionaries.""" + base = {'level1': {'level2': {'key1': 'value1', 'key2': 'value2'}}} + override = {'level1': {'level2': {'key2': 'override2', 'key3': 'value3'}}} + result = deep_merge(base, override) + + assert result == {'level1': {'level2': {'key1': 'value1', 'key2': 'override2', 'key3': 'value3'}}} + + +def test_deep_merge_override_with_non_dict() -> None: + """Test that non-dict overrides replace the base value entirely.""" + base = {'key': {'nested': 'value'}} + override = {'key': 'simple_value'} + result = deep_merge(base, override) + + assert result == {'key': 'simple_value'} + + +def test_load_yaml_file_nonexistent(fs: FakeFilesystem) -> None: + """Test loading a non-existent file returns None.""" + result = load_yaml_file(Path('/fake/nonexistent.yaml')) + assert result is None + + +def test_load_yaml_file_valid_yaml(fs: FakeFilesystem) -> None: + """Test loading a valid YAML file.""" + fs.create_file('/fake/config.yaml', contents='mode: block\nfail_open: true\n') + + result = load_yaml_file(Path('/fake/config.yaml')) + assert result == {'mode': 'block', 'fail_open': True} + + +def test_load_yaml_file_valid_json(fs: FakeFilesystem) -> None: + """Test loading a valid JSON file.""" + fs.create_file('/fake/config.json', contents='{"mode": "block", "fail_open": true}') + + result = load_yaml_file(Path('/fake/config.json')) + assert result == {'mode': 'block', 'fail_open': True} + + +def test_load_yaml_file_invalid_yaml(fs: FakeFilesystem) -> None: + """Test loading an invalid YAML file returns None.""" + fs.create_file('/fake/invalid.yaml', contents='{ invalid yaml content [') + + result = load_yaml_file(Path('/fake/invalid.yaml')) + assert result is None + + +def test_load_defaults() -> None: + """Test that load_defaults returns a dict with expected keys.""" + defaults = load_defaults() + + assert isinstance(defaults, dict) + assert 'mode' in defaults + assert 'fail_open' in defaults + assert 'prompt' in defaults + assert 'file_read' in defaults + assert 'mcp' in defaults + + +def test_get_policy_value_single_key() -> None: + """Test getting a single-level value.""" + policy = {'mode': 'block', 'fail_open': True} + + assert get_policy_value(policy, 'mode') == 'block' + assert get_policy_value(policy, 'fail_open') is True + + +def test_get_policy_value_nested_keys() -> None: + """Test getting a nested value.""" + policy = {'prompt': {'enabled': True, 'action': 'block'}} + + assert get_policy_value(policy, 'prompt', 'enabled') is True + assert get_policy_value(policy, 'prompt', 'action') == 'block' + + +def test_get_policy_value_missing_key() -> None: + """Test that missing keys return the default value.""" + policy = {'mode': 'block'} + + assert get_policy_value(policy, 'nonexistent', default='default_value') == 'default_value' + + +def test_get_policy_value_deeply_nested() -> None: + """Test getting deeply nested values.""" + policy = {'level1': {'level2': {'level3': 'value'}}} + + assert get_policy_value(policy, 'level1', 'level2', 'level3') == 'value' + assert get_policy_value(policy, 'level1', 'level2', 'missing', default='def') == 'def' + + +def test_get_policy_value_non_dict_in_path() -> None: + """Test that non-dict values in path return default.""" + policy = {'key': 'string_value'} + + # Trying to access nested key on non-dict should return default + assert get_policy_value(policy, 'key', 'nested', default='default') == 'default' + + +@patch('cycode.cli.apps.ai_guardrails.scan.policy.load_yaml_file') +def test_load_policy_defaults_only(mock_load: MagicMock) -> None: + """Test loading policy with only defaults (no user or repo config).""" + mock_load.return_value = None # No user or repo config + + policy = load_policy() + + assert 'mode' in policy + assert 'fail_open' in policy + + +@patch('pathlib.Path.home') +def test_load_policy_with_user_config(mock_home: MagicMock, fs: FakeFilesystem) -> None: + """Test loading policy with user config override.""" + mock_home.return_value = Path('/home/testuser') + + # Create user config in fake filesystem + fs.create_file('/home/testuser/.cycode/ai-guardrails.yaml', contents='mode: warn\nfail_open: false\n') + + policy = load_policy() + + # User config should override defaults + assert policy['mode'] == 'warn' + assert policy['fail_open'] is False + + +@patch('cycode.cli.apps.ai_guardrails.scan.policy.load_yaml_file') +def test_load_policy_with_repo_config(mock_load: MagicMock) -> None: + """Test loading policy with repo config (highest precedence).""" + repo_path = Path('/fake/repo') + repo_config = repo_path / '.cycode' / 'ai-guardrails.yaml' + + def side_effect(path: Path) -> Optional[dict]: + if path == repo_config: + return {'mode': 'block', 'prompt': {'enabled': False}} + return None + + mock_load.side_effect = side_effect + + policy = load_policy(str(repo_path)) + + # Repo config should have highest precedence + assert policy['mode'] == 'block' + assert policy['prompt']['enabled'] is False + + +@patch('pathlib.Path.home') +def test_load_policy_precedence(mock_home: MagicMock, fs: FakeFilesystem) -> None: + """Test that policy precedence is: defaults < user < repo.""" + mock_home.return_value = Path('/home/testuser') + + # Create user config + fs.create_file('/home/testuser/.cycode/ai-guardrails.yaml', contents='mode: warn\nfail_open: false\n') + + # Create repo config + fs.create_file('/fake/repo/.cycode/ai-guardrails.yaml', contents='mode: block\n') + + policy = load_policy('/fake/repo') + + # mode should come from repo (highest precedence) + assert policy['mode'] == 'block' + # fail_open should come from user config (repo doesn't override it) + assert policy['fail_open'] is False + + +@patch('cycode.cli.apps.ai_guardrails.scan.policy.load_yaml_file') +def test_load_policy_none_workspace_root(mock_load: MagicMock) -> None: + """Test that None workspace_root is handled correctly.""" + mock_load.return_value = None + + policy = load_policy(None) + + # Should only load defaults (no repo config) + assert 'mode' in policy diff --git a/tests/cli/commands/ai_guardrails/scan/test_response_builders.py b/tests/cli/commands/ai_guardrails/scan/test_response_builders.py new file mode 100644 index 00000000..45f80829 --- /dev/null +++ b/tests/cli/commands/ai_guardrails/scan/test_response_builders.py @@ -0,0 +1,148 @@ +"""Tests for IDE response builders.""" + +import pytest + +from cycode.cli.apps.ai_guardrails.scan.response_builders import ( + ClaudeCodeResponseBuilder, + CursorResponseBuilder, + IDEResponseBuilder, + get_response_builder, +) + + +def test_cursor_response_builder_allow_permission() -> None: + """Test Cursor allow permission response.""" + builder = CursorResponseBuilder() + response = builder.allow_permission() + + assert response == {'permission': 'allow'} + + +def test_cursor_response_builder_deny_permission() -> None: + """Test Cursor deny permission response with messages.""" + builder = CursorResponseBuilder() + response = builder.deny_permission('User message', 'Agent message') + + assert response == { + 'permission': 'deny', + 'user_message': 'User message', + 'agent_message': 'Agent message', + } + + +def test_cursor_response_builder_ask_permission() -> None: + """Test Cursor ask permission response for warnings.""" + builder = CursorResponseBuilder() + response = builder.ask_permission('Warning message', 'Agent warning') + + assert response == { + 'permission': 'ask', + 'user_message': 'Warning message', + 'agent_message': 'Agent warning', + } + + +def test_cursor_response_builder_allow_prompt() -> None: + """Test Cursor allow prompt response.""" + builder = CursorResponseBuilder() + response = builder.allow_prompt() + + assert response == {'continue': True} + + +def test_cursor_response_builder_deny_prompt() -> None: + """Test Cursor deny prompt response with message.""" + builder = CursorResponseBuilder() + response = builder.deny_prompt('Secrets detected') + + assert response == {'continue': False, 'user_message': 'Secrets detected'} + + +def test_get_response_builder_cursor() -> None: + """Test getting Cursor response builder.""" + builder = get_response_builder('cursor') + + assert isinstance(builder, CursorResponseBuilder) + assert isinstance(builder, IDEResponseBuilder) + + +def test_get_response_builder_unsupported() -> None: + """Test that unsupported IDE raises ValueError.""" + with pytest.raises(ValueError, match='Unsupported IDE: unknown'): + get_response_builder('unknown') + + +def test_cursor_response_builder_is_singleton() -> None: + """Test that getting the same builder returns the same instance.""" + builder1 = get_response_builder('cursor') + builder2 = get_response_builder('cursor') + + assert builder1 is builder2 + + +# Claude Code response builder tests + + +def test_claude_code_response_builder_allow_permission() -> None: + """Test Claude Code allow permission response.""" + builder = ClaudeCodeResponseBuilder() + response = builder.allow_permission() + + assert response == { + 'hookSpecificOutput': { + 'hookEventName': 'PreToolUse', + 'permissionDecision': 'allow', + } + } + + +def test_claude_code_response_builder_deny_permission() -> None: + """Test Claude Code deny permission response with messages.""" + builder = ClaudeCodeResponseBuilder() + response = builder.deny_permission('User message', 'Agent message') + + assert response == { + 'hookSpecificOutput': { + 'hookEventName': 'PreToolUse', + 'permissionDecision': 'deny', + 'permissionDecisionReason': 'User message', + } + } + + +def test_claude_code_response_builder_ask_permission() -> None: + """Test Claude Code ask permission response for warnings.""" + builder = ClaudeCodeResponseBuilder() + response = builder.ask_permission('Warning message', 'Agent warning') + + assert response == { + 'hookSpecificOutput': { + 'hookEventName': 'PreToolUse', + 'permissionDecision': 'ask', + 'permissionDecisionReason': 'Warning message', + } + } + + +def test_claude_code_response_builder_allow_prompt() -> None: + """Test Claude Code allow prompt response (empty dict).""" + builder = ClaudeCodeResponseBuilder() + response = builder.allow_prompt() + + assert response == {} + + +def test_claude_code_response_builder_deny_prompt() -> None: + """Test Claude Code deny prompt response with message.""" + builder = ClaudeCodeResponseBuilder() + response = builder.deny_prompt('Secrets detected') + + assert response == {'decision': 'block', 'reason': 'Secrets detected'} + + +def test_get_response_builder_claude_code() -> None: + """Test getting Claude Code response builder.""" + builder = get_response_builder('claude-code') + + assert isinstance(builder, ClaudeCodeResponseBuilder) + assert isinstance(builder, IDEResponseBuilder) diff --git a/tests/cli/commands/ai_guardrails/scan/test_scan_command.py b/tests/cli/commands/ai_guardrails/scan/test_scan_command.py new file mode 100644 index 00000000..4bcb35f2 --- /dev/null +++ b/tests/cli/commands/ai_guardrails/scan/test_scan_command.py @@ -0,0 +1,169 @@ +"""Tests for AI guardrails scan command.""" + +import json +from io import StringIO +from unittest.mock import MagicMock + +import pytest +from pytest_mock import MockerFixture +from typer.testing import CliRunner + +from cycode.cli.apps.ai_guardrails import app as ai_guardrails_app +from cycode.cli.apps.ai_guardrails.scan.scan_command import scan_command + + +@pytest.fixture +def mock_ctx() -> MagicMock: + """Create a mock typer context.""" + ctx = MagicMock() + ctx.obj = {} + return ctx + + +@pytest.fixture +def mock_scan_command_deps(mocker: MockerFixture) -> dict[str, MagicMock]: + """Mock scan_command dependencies that should not be called on early exit.""" + return { + 'initialize_clients': mocker.patch('cycode.cli.apps.ai_guardrails.scan.scan_command._initialize_clients'), + 'load_policy': mocker.patch('cycode.cli.apps.ai_guardrails.scan.scan_command.load_policy'), + 'get_handler': mocker.patch('cycode.cli.apps.ai_guardrails.scan.scan_command.get_handler_for_event'), + } + + +def _assert_no_api_calls(mocks: dict[str, MagicMock]) -> None: + """Assert that no API-related functions were called.""" + mocks['initialize_clients'].assert_not_called() + mocks['load_policy'].assert_not_called() + mocks['get_handler'].assert_not_called() + + +class TestIdeMismatchSkipsProcessing: + """Tests that verify IDE mismatch causes early exit without API calls.""" + + def test_claude_code_payload_with_cursor_ide( + self, + mock_ctx: MagicMock, + mocker: MockerFixture, + capsys: pytest.CaptureFixture[str], + mock_scan_command_deps: dict[str, MagicMock], + ) -> None: + """Test Claude Code payload is skipped when --ide cursor is specified. + + When Cursor reads Claude Code hooks from ~/.claude/settings.json, it will invoke + the hook with Claude Code event names. The scan command should skip processing. + """ + payload = {'hook_event_name': 'UserPromptSubmit', 'session_id': 'session-123', 'prompt': 'test'} + mocker.patch('sys.stdin', StringIO(json.dumps(payload))) + + scan_command(mock_ctx, ide='cursor') + + _assert_no_api_calls(mock_scan_command_deps) + response = json.loads(capsys.readouterr().out) + assert response.get('continue') is True + + def test_cursor_payload_with_claude_code_ide( + self, + mock_ctx: MagicMock, + mocker: MockerFixture, + capsys: pytest.CaptureFixture[str], + mock_scan_command_deps: dict[str, MagicMock], + ) -> None: + """Test Cursor payload is skipped when --ide claude-code is specified.""" + payload = {'hook_event_name': 'beforeSubmitPrompt', 'conversation_id': 'conv-123', 'prompt': 'test'} + mocker.patch('sys.stdin', StringIO(json.dumps(payload))) + + scan_command(mock_ctx, ide='claude-code') + + _assert_no_api_calls(mock_scan_command_deps) + response = json.loads(capsys.readouterr().out) + assert response == {} # Claude Code allow_prompt returns empty dict + + +class TestInvalidPayloadSkipsProcessing: + """Tests that verify invalid payloads cause early exit without API calls.""" + + def test_empty_payload( + self, + mock_ctx: MagicMock, + mocker: MockerFixture, + capsys: pytest.CaptureFixture[str], + mock_scan_command_deps: dict[str, MagicMock], + ) -> None: + """Test empty payload skips processing.""" + mocker.patch('sys.stdin', StringIO('')) + + scan_command(mock_ctx, ide='cursor') + + mock_scan_command_deps['initialize_clients'].assert_not_called() + response = json.loads(capsys.readouterr().out) + assert response.get('continue') is True + + def test_invalid_json_payload( + self, + mock_ctx: MagicMock, + mocker: MockerFixture, + capsys: pytest.CaptureFixture[str], + mock_scan_command_deps: dict[str, MagicMock], + ) -> None: + """Test invalid JSON skips processing.""" + mocker.patch('sys.stdin', StringIO('not valid json {')) + + scan_command(mock_ctx, ide='cursor') + + mock_scan_command_deps['initialize_clients'].assert_not_called() + response = json.loads(capsys.readouterr().out) + assert response.get('continue') is True + + +class TestMatchingIdeProcessesPayload: + """Tests that verify matching IDE processes the payload normally.""" + + def test_claude_code_payload_with_claude_code_ide( + self, + mock_ctx: MagicMock, + mocker: MockerFixture, + mock_scan_command_deps: dict[str, MagicMock], + ) -> None: + """Test Claude Code payload is processed when --ide claude-code is specified.""" + payload = {'hook_event_name': 'UserPromptSubmit', 'session_id': 'session-123', 'prompt': 'test'} + mocker.patch('sys.stdin', StringIO(json.dumps(payload))) + + mock_scan_command_deps['load_policy'].return_value = {'fail_open': True} + mock_handler = MagicMock(return_value={'decision': 'allow'}) + mock_scan_command_deps['get_handler'].return_value = mock_handler + + scan_command(mock_ctx, ide='claude-code') + + mock_scan_command_deps['initialize_clients'].assert_called_once() + mock_scan_command_deps['load_policy'].assert_called_once() + mock_scan_command_deps['get_handler'].assert_called_once() + mock_handler.assert_called_once() + + +class TestDefaultIdeParameterViaCli: + """Tests that verify default IDE parameter works correctly via CLI invocation.""" + + def test_scan_command_default_ide_via_cli(self, mocker: MockerFixture) -> None: + """Test scan_command works with default --ide when invoked via CLI. + + This test catches issues where Typer converts enum defaults to strings + incorrectly (e.g., AIIDEType.CURSOR becomes 'AIIDEType.CURSOR' instead of 'cursor'). + """ + mocker.patch('cycode.cli.apps.ai_guardrails.scan.scan_command._initialize_clients') + mocker.patch( + 'cycode.cli.apps.ai_guardrails.scan.scan_command.load_policy', + return_value={'fail_open': True}, + ) + mock_handler = MagicMock(return_value={'continue': True}) + mocker.patch( + 'cycode.cli.apps.ai_guardrails.scan.scan_command.get_handler_for_event', + return_value=mock_handler, + ) + + runner = CliRunner() + payload = json.dumps({'hook_event_name': 'beforeSubmitPrompt', 'prompt': 'test'}) + + # Invoke via CLI without --ide flag to use default + result = runner.invoke(ai_guardrails_app, ['scan'], input=payload) + + assert result.exit_code == 0, f'Command failed: {result.output}' diff --git a/tests/cli/commands/ai_guardrails/scan/test_utils.py b/tests/cli/commands/ai_guardrails/scan/test_utils.py new file mode 100644 index 00000000..ce84c609 --- /dev/null +++ b/tests/cli/commands/ai_guardrails/scan/test_utils.py @@ -0,0 +1,113 @@ +"""Tests for AI guardrails utility functions.""" + +from cycode.cli.apps.ai_guardrails.scan.utils import ( + is_denied_path, + matches_glob, + normalize_path, +) + + +def test_normalize_path_rejects_escape() -> None: + """Test that paths attempting to escape are rejected.""" + path = '../../../etc/passwd' + result = normalize_path(path) + + assert result == '' + + +def test_normalize_path_empty() -> None: + """Test normalizing empty path.""" + result = normalize_path('') + + assert result == '' + + +def test_matches_glob_simple() -> None: + """Test simple glob pattern matching.""" + assert matches_glob('secret.env', '*.env') is True + assert matches_glob('secret.txt', '*.env') is False + + +def test_matches_glob_recursive() -> None: + """Test recursive glob pattern with **.""" + assert matches_glob('path/to/secret.env', '**/*.env') is True + # Note: '**/*.env' requires at least one path separator, so 'secret.env' won't match + assert matches_glob('secret.env', '*.env') is True # Use non-recursive pattern instead + assert matches_glob('path/to/file.txt', '**/*.env') is False + + +def test_matches_glob_directory() -> None: + """Test matching files in specific directories.""" + assert matches_glob('.env', '.env') is True + assert matches_glob('config/.env', '**/.env') is True + assert matches_glob('other/file', '**/.env') is False + + +def test_matches_glob_case_insensitive() -> None: + """Test that glob matching handles case variations.""" + # Case-insensitive matching for cross-platform compatibility + assert matches_glob('secret.env', '*.env') is True + assert matches_glob('SECRET.ENV', '*.env') is True # Uppercase path matches lowercase pattern + assert matches_glob('Secret.Env', '*.env') is True # Mixed case matches + assert matches_glob('secret.env', '*.ENV') is True # Lowercase path matches uppercase pattern + assert matches_glob('SECRET.ENV', '*.ENV') is True # Both uppercase match + + +def test_matches_glob_empty_inputs() -> None: + """Test glob matching with empty inputs.""" + assert matches_glob('', '*.env') is False + assert matches_glob('file.env', '') is False + assert matches_glob('', '') is False + + +def test_matches_glob_with_traversal_attempt() -> None: + """Test that path traversal is normalized before matching.""" + # Path traversal attempts should be normalized + assert matches_glob('../secret.env', '*.env') is False + + +def test_is_denied_path_with_deny_globs() -> None: + """Test path denial with deny_globs policy.""" + policy = {'file_read': {'deny_globs': ['*.env', '.git/*', '**/secrets/*']}} + + assert is_denied_path('.env', policy) is True + # Note: Path.match('*.env') matches paths ending with .env, including nested paths + assert is_denied_path('config/.env', policy) is True # Matches *.env + assert is_denied_path('.git/config', policy) is True # Matches .git/* + assert is_denied_path('app/secrets/api_keys.txt', policy) is True # Matches **/secrets/* + assert is_denied_path('app/config.yaml', policy) is False + + +def test_is_denied_path_nested_patterns() -> None: + """Test denial with various nesting patterns.""" + policy = {'file_read': {'deny_globs': ['*.key', '**/*.key', 'config/*.env']}} + + # *.key matches .key files at root level, **/*.key for nested + assert is_denied_path('private.key', policy) is True + assert is_denied_path('app/private.key', policy) is True + # config/*.env only matches .env files directly in config/ + assert is_denied_path('config/app.env', policy) is True + assert is_denied_path('config/sub/app.env', policy) is False # Not direct child + assert is_denied_path('app/config.yaml', policy) is False + + +def test_is_denied_path_empty_globs() -> None: + """Test that empty deny_globs list denies nothing.""" + policy = {'file_read': {'deny_globs': []}} + + assert is_denied_path('.env', policy) is False + assert is_denied_path('any/path', policy) is False + + +def test_is_denied_path_no_policy() -> None: + """Test denial with missing policy configuration.""" + policy = {} + + assert is_denied_path('.env', policy) is False + + +def test_is_denied_path_empty_path() -> None: + """Test denial check with empty path.""" + policy = {'file_read': {'deny_globs': ['*.env']}} + + assert is_denied_path('', policy) is False diff --git a/tests/cli/commands/ai_guardrails/test_command_utils.py b/tests/cli/commands/ai_guardrails/test_command_utils.py new file mode 100644 index 00000000..5d8d224b --- /dev/null +++ b/tests/cli/commands/ai_guardrails/test_command_utils.py @@ -0,0 +1,60 @@ +"""Tests for AI guardrails command utilities.""" + +import pytest +import typer + +from cycode.cli.apps.ai_guardrails.command_utils import ( + validate_and_parse_ide, + validate_scope, +) +from cycode.cli.apps.ai_guardrails.consts import AIIDEType + + +def test_validate_and_parse_ide_valid() -> None: + """Test parsing valid IDE names.""" + assert validate_and_parse_ide('cursor') == AIIDEType.CURSOR + assert validate_and_parse_ide('CURSOR') == AIIDEType.CURSOR + assert validate_and_parse_ide('CuRsOr') == AIIDEType.CURSOR + assert validate_and_parse_ide('claude-code') == AIIDEType.CLAUDE_CODE + assert validate_and_parse_ide('Claude-Code') == AIIDEType.CLAUDE_CODE + assert validate_and_parse_ide('all') is None + + +def test_validate_and_parse_ide_invalid() -> None: + """Test that invalid IDE raises typer.Exit.""" + with pytest.raises(typer.Exit) as exc_info: + validate_and_parse_ide('invalid_ide') + assert exc_info.value.exit_code == 1 + + +def test_validate_scope_valid_default() -> None: + """Test validating valid scope with default allowed scopes.""" + # Should not raise any exception + validate_scope('user') + validate_scope('repo') + + +def test_validate_scope_invalid_default() -> None: + """Test that invalid scope raises typer.Exit with default allowed scopes.""" + with pytest.raises(typer.Exit) as exc_info: + validate_scope('invalid') + assert exc_info.value.exit_code == 1 + + with pytest.raises(typer.Exit) as exc_info: + validate_scope('all') # 'all' not in default allowed scopes + assert exc_info.value.exit_code == 1 + + +def test_validate_scope_valid_custom() -> None: + """Test validating scope with custom allowed scopes.""" + # Should not raise any exception + validate_scope('user', allowed_scopes=('user', 'repo', 'all')) + validate_scope('repo', allowed_scopes=('user', 'repo', 'all')) + validate_scope('all', allowed_scopes=('user', 'repo', 'all')) + + +def test_validate_scope_invalid_custom() -> None: + """Test that invalid scope raises typer.Exit with custom allowed scopes.""" + with pytest.raises(typer.Exit) as exc_info: + validate_scope('invalid', allowed_scopes=('user', 'repo', 'all')) + assert exc_info.value.exit_code == 1 diff --git a/tests/cli/commands/ai_guardrails/test_hooks_manager.py b/tests/cli/commands/ai_guardrails/test_hooks_manager.py new file mode 100644 index 00000000..f0dec6f7 --- /dev/null +++ b/tests/cli/commands/ai_guardrails/test_hooks_manager.py @@ -0,0 +1,53 @@ +"""Tests for AI guardrails hooks manager.""" + +from cycode.cli.apps.ai_guardrails.hooks_manager import is_cycode_hook_entry + + +def test_is_cycode_hook_entry_cursor_format() -> None: + """Test detecting Cycode hook in Cursor format (flat command).""" + entry = {'command': 'cycode ai-guardrails scan'} + assert is_cycode_hook_entry(entry) is True + + entry = {'command': 'cycode ai-guardrails scan --some-flag'} + assert is_cycode_hook_entry(entry) is True + + +def test_is_cycode_hook_entry_claude_code_format() -> None: + """Test detecting Cycode hook in Claude Code format (nested).""" + entry = { + 'hooks': [{'type': 'command', 'command': 'cycode ai-guardrails scan --ide claude-code'}], + } + assert is_cycode_hook_entry(entry) is True + + entry = { + 'matcher': 'Read', + 'hooks': [{'type': 'command', 'command': 'cycode ai-guardrails scan --ide claude-code'}], + } + assert is_cycode_hook_entry(entry) is True + + +def test_is_cycode_hook_entry_non_cycode() -> None: + """Test that non-Cycode hooks are not detected.""" + # Cursor format + entry = {'command': 'some-other-command'} + assert is_cycode_hook_entry(entry) is False + + # Claude Code format + entry = { + 'hooks': [{'type': 'command', 'command': 'some-other-command'}], + } + assert is_cycode_hook_entry(entry) is False + + # Empty entry + entry = {} + assert is_cycode_hook_entry(entry) is False + + +def test_is_cycode_hook_entry_partial_match() -> None: + """Test partial command match.""" + # Should match if command contains 'cycode ai-guardrails scan' + entry = {'command': '/usr/local/bin/cycode ai-guardrails scan'} + assert is_cycode_hook_entry(entry) is True + + entry = {'command': 'cycode ai-guardrails scan --verbose'} + assert is_cycode_hook_entry(entry) is True diff --git a/tests/cli/commands/configure/__init__.py b/tests/cli/commands/configure/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cli/commands/configure/test_configure_command.py b/tests/cli/commands/configure/test_configure_command.py new file mode 100644 index 00000000..0d763edd --- /dev/null +++ b/tests/cli/commands/configure/test_configure_command.py @@ -0,0 +1,314 @@ +from typing import TYPE_CHECKING + +from typer.testing import CliRunner + +from cycode.cli.app import app + +if TYPE_CHECKING: + from pytest_mock import MockerFixture + + +def test_configure_command_no_exist_values_in_file(mocker: 'MockerFixture') -> None: + # Arrange + app_url_user_input = 'new app url' + api_url_user_input = 'new api url' + client_id_user_input = 'new client id' + client_secret_user_input = 'new client secret' + id_token_user_input = 'new id token' + + mocker.patch( + 'cycode.cli.user_settings.credentials_manager.CredentialsManager.get_credentials_from_file', + return_value=(None, None), + ) + mocker.patch( + 'cycode.cli.user_settings.credentials_manager.CredentialsManager.get_oidc_credentials_from_file', + return_value=(None, None), + ) + mocker.patch( + 'cycode.cli.user_settings.config_file_manager.ConfigFileManager.get_api_url', + return_value=None, + ) + mocker.patch( + 'cycode.cli.user_settings.config_file_manager.ConfigFileManager.get_app_url', + return_value=None, + ) + + # side effect - multiple return values, each item in the list represents return of a call + mocker.patch( + 'typer.prompt', + side_effect=[ + api_url_user_input, + app_url_user_input, + client_id_user_input, + client_secret_user_input, + id_token_user_input, + ], + ) + + mocked_update_credentials = mocker.patch( + 'cycode.cli.user_settings.credentials_manager.CredentialsManager.update_credentials' + ) + mocked_update_oidc_credentials = mocker.patch( + 'cycode.cli.user_settings.credentials_manager.CredentialsManager.update_oidc_credentials' + ) + mocked_update_api_base_url = mocker.patch( + 'cycode.cli.user_settings.config_file_manager.ConfigFileManager.update_api_base_url' + ) + mocked_update_app_base_url = mocker.patch( + 'cycode.cli.user_settings.config_file_manager.ConfigFileManager.update_app_base_url' + ) + + # Act + CliRunner().invoke(app, ['configure']) + + # Assert + mocked_update_credentials.assert_called_once_with(client_id_user_input, client_secret_user_input) + mocked_update_oidc_credentials.assert_called_once_with(client_id_user_input, id_token_user_input) + mocked_update_api_base_url.assert_called_once_with(api_url_user_input) + mocked_update_app_base_url.assert_called_once_with(app_url_user_input) + + +def test_configure_command_update_current_configs_in_files(mocker: 'MockerFixture') -> None: + # Arrange + app_url_user_input = 'new app url' + api_url_user_input = 'new api url' + client_id_user_input = 'new client id' + client_secret_user_input = 'new client secret' + id_token_user_input = 'new id token' + + mocker.patch( + 'cycode.cli.user_settings.credentials_manager.CredentialsManager.get_credentials_from_file', + return_value=('client id file', 'client secret file'), + ) + mocker.patch( + 'cycode.cli.user_settings.credentials_manager.CredentialsManager.get_oidc_credentials_from_file', + return_value=('client id file', 'id token file'), + ) + mocker.patch( + 'cycode.cli.user_settings.config_file_manager.ConfigFileManager.get_api_url', + return_value='api url file', + ) + mocker.patch( + 'cycode.cli.user_settings.config_file_manager.ConfigFileManager.get_app_url', + return_value='app url file', + ) + + # side effect - multiple return values, each item in the list represents return of a call + mocker.patch( + 'typer.prompt', + side_effect=[ + api_url_user_input, + app_url_user_input, + client_id_user_input, + client_secret_user_input, + id_token_user_input, + ], + ) + + mocked_update_credentials = mocker.patch( + 'cycode.cli.user_settings.credentials_manager.CredentialsManager.update_credentials' + ) + mocked_update_api_base_url = mocker.patch( + 'cycode.cli.user_settings.config_file_manager.ConfigFileManager.update_api_base_url' + ) + mocked_update_app_base_url = mocker.patch( + 'cycode.cli.user_settings.config_file_manager.ConfigFileManager.update_app_base_url' + ) + mocker_update_oidc_credentials = mocker.patch( + 'cycode.cli.user_settings.credentials_manager.CredentialsManager.update_oidc_credentials' + ) + + # Act + CliRunner().invoke(app, ['configure']) + + # Assert + mocked_update_credentials.assert_called_once_with(client_id_user_input, client_secret_user_input) + mocker_update_oidc_credentials.assert_called_once_with(client_id_user_input, id_token_user_input) + mocked_update_api_base_url.assert_called_once_with(api_url_user_input) + mocked_update_app_base_url.assert_called_once_with(app_url_user_input) + + +def test_set_credentials_update_only_client_id(mocker: 'MockerFixture') -> None: + # Arrange + client_id_user_input = 'new client id' + current_client_id = 'client secret file' + mocker.patch( + 'cycode.cli.user_settings.credentials_manager.CredentialsManager.get_credentials_from_file', + return_value=('client id file', 'client secret file'), + ) + + # side effect - multiple return values, each item in the list represents return of a call + mocker.patch('typer.prompt', side_effect=['', '', client_id_user_input, '', '']) + mocked_update_credentials = mocker.patch( + 'cycode.cli.user_settings.credentials_manager.CredentialsManager.update_credentials' + ) + + # Act + CliRunner().invoke(app, ['configure']) + + # Assert + mocked_update_credentials.assert_called_once_with(client_id_user_input, current_client_id) + + +def test_configure_command_update_only_client_secret(mocker: 'MockerFixture') -> None: + # Arrange + client_secret_user_input = 'new client secret' + current_client_id = 'client secret file' + + mocker.patch( + 'cycode.cli.user_settings.credentials_manager.CredentialsManager.get_credentials_from_file', + return_value=(current_client_id, 'client secret file'), + ) + + # side effect - multiple return values, each item in the list represents return of a call + mocker.patch('typer.prompt', side_effect=['', '', '', client_secret_user_input, '']) + mocked_update_credentials = mocker.patch( + 'cycode.cli.user_settings.credentials_manager.CredentialsManager.update_credentials' + ) + + # Act + CliRunner().invoke(app, ['configure']) + + # Assert + mocked_update_credentials.assert_called_once_with(current_client_id, client_secret_user_input) + + +def test_configure_command_update_only_api_url(mocker: 'MockerFixture') -> None: + # Arrange + api_url_user_input = 'new api url' + current_api_url = 'api url' + + mocker.patch( + 'cycode.cli.user_settings.config_file_manager.ConfigFileManager.get_api_url', + return_value=current_api_url, + ) + + # side effect - multiple return values, each item in the list represents return of a call + mocker.patch('typer.prompt', side_effect=[api_url_user_input, '', '', '', '']) + mocked_update_api_base_url = mocker.patch( + 'cycode.cli.user_settings.config_file_manager.ConfigFileManager.update_api_base_url' + ) + + # Act + CliRunner().invoke(app, ['configure']) + + # Assert + mocked_update_api_base_url.assert_called_once_with(api_url_user_input) + + +def test_configure_command_update_only_id_token(mocker: 'MockerFixture') -> None: + # Arrange + current_client_id = 'client id file' + current_id_token = 'old id token' + new_id_token = 'new id token' + + mocker.patch( + 'cycode.cli.user_settings.credentials_manager.CredentialsManager.get_credentials_from_file', + return_value=(current_client_id, 'client secret file'), + ) + mocker.patch( + 'cycode.cli.user_settings.credentials_manager.CredentialsManager.get_oidc_credentials_from_file', + return_value=(current_client_id, current_id_token), + ) + + mocker.patch('typer.prompt', side_effect=['', '', '', '', new_id_token]) + + mocked_update_oidc_credentials = mocker.patch( + 'cycode.cli.user_settings.credentials_manager.CredentialsManager.update_oidc_credentials' + ) + + # Act + CliRunner().invoke(app, ['configure']) + + # Assert + mocked_update_oidc_credentials.assert_called_once_with(current_client_id, new_id_token) + + +def test_configure_command_should_not_update_credentials(mocker: 'MockerFixture') -> None: + # Arrange + client_id_user_input = '' + client_secret_user_input = '' + + mocker.patch( + 'cycode.cli.user_settings.credentials_manager.CredentialsManager.get_credentials_from_file', + return_value=('client id file', 'client secret file'), + ) + + # side effect - multiple return values, each item in the list represents return of a call + mocker.patch('typer.prompt', side_effect=['', '', client_id_user_input, client_secret_user_input, '']) + mocked_update_credentials = mocker.patch( + 'cycode.cli.user_settings.credentials_manager.CredentialsManager.update_credentials' + ) + + # Act + CliRunner().invoke(app, ['configure']) + + # Assert + assert not mocked_update_credentials.called + + +def test_configure_command_should_not_update_config_file(mocker: 'MockerFixture') -> None: + # Arrange + app_url_user_input = '' + api_url_user_input = '' + + mocker.patch( + 'cycode.cli.user_settings.config_file_manager.ConfigFileManager.get_api_url', + return_value='api url file', + ) + mocker.patch( + 'cycode.cli.user_settings.config_file_manager.ConfigFileManager.get_app_url', + return_value='app url file', + ) + + # side effect - multiple return values, each item in the list represents return of a call + mocker.patch('typer.prompt', side_effect=[api_url_user_input, app_url_user_input, '', '', '']) + mocked_update_api_base_url = mocker.patch( + 'cycode.cli.user_settings.config_file_manager.ConfigFileManager.update_api_base_url' + ) + mocked_update_app_base_url = mocker.patch( + 'cycode.cli.user_settings.config_file_manager.ConfigFileManager.update_app_base_url' + ) + + # Act + CliRunner().invoke(app, ['configure']) + + # Assert + assert not mocked_update_api_base_url.called + assert not mocked_update_app_base_url.called + + +def test_configure_command_should_not_update_oidc_credentials(mocker: 'MockerFixture') -> None: + # Arrange + current_client_id = 'client id file' + current_client_secret = 'client secret file' + current_id_token = 'old id token' + + mocker.patch( + 'cycode.cli.user_settings.credentials_manager.CredentialsManager.get_credentials_from_file', + return_value=(current_client_id, current_client_secret), + ) + mocker.patch( + 'cycode.cli.user_settings.credentials_manager.CredentialsManager.get_oidc_credentials_from_file', + return_value=(current_client_id, current_id_token), + ) + mocker.patch( + 'cycode.cli.user_settings.config_file_manager.ConfigFileManager.get_api_url', + return_value='api url file', + ) + mocker.patch( + 'cycode.cli.user_settings.config_file_manager.ConfigFileManager.get_app_url', + return_value='app url file', + ) + + mocker.patch('typer.prompt', side_effect=['', '', '', '', '']) + + mocked_update_oidc_credentials = mocker.patch( + 'cycode.cli.user_settings.credentials_manager.CredentialsManager.update_oidc_credentials' + ) + + # Act + CliRunner().invoke(app, ['configure']) + + # Assert + mocked_update_oidc_credentials.assert_not_called() diff --git a/tests/cli/commands/import/__init__.py b/tests/cli/commands/import/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cli/commands/import/test_import_sbom.py b/tests/cli/commands/import/test_import_sbom.py new file mode 100644 index 00000000..ecea2c6d --- /dev/null +++ b/tests/cli/commands/import/test_import_sbom.py @@ -0,0 +1,329 @@ +import tempfile +from pathlib import Path +from typing import TYPE_CHECKING + +import pytest +import responses +from typer.testing import CliRunner + +from cycode.cli.app import app +from cycode.cli.cli_types import BusinessImpactOption +from cycode.cyclient.client_creator import create_import_sbom_client +from cycode.cyclient.import_sbom_client import ImportSbomClient, ImportSbomParameters +from tests.conftest import _CLIENT_ID, _CLIENT_SECRET, CLI_ENV_VARS +from tests.cyclient.mocked_responses import import_sbom_client as mocked_import_sbom + +if TYPE_CHECKING: + from pytest_mock import MockerFixture + + +@pytest.fixture(scope='session') +def import_sbom_client() -> ImportSbomClient: + return create_import_sbom_client(_CLIENT_ID, _CLIENT_SECRET, False) + + +def _validate_called_endpoint(calls: responses.CallList, path: str, expected_count: int = 1) -> None: + # Verify the import request was made + import_calls = [c for c in calls if path in c.request.url] + assert len(import_calls) == expected_count + + +class TestImportSbomParameters: + """Tests for ImportSbomParameters.to_request_form() method""" + + def test_to_request_form_with_all_fields(self) -> None: + data = { + 'Name': 'test-sbom', + 'Vendor': 'test-vendor', + 'BusinessImpact': BusinessImpactOption.HIGH, + 'Labels': ['label1', 'label2'], + 'Owners': ['owner1-id', 'owner2-id'], + } + + params = ImportSbomParameters(**data) + form_data = params.to_request_form() + + for key, val in data.items(): + if isinstance(val, list): + assert form_data[f'{key}[]'] == val + else: + assert form_data[key] == val + + def test_to_request_form_with_required_fields_only(self) -> None: + params = ImportSbomParameters( + Name='test-sbom', + Vendor='test-vendor', + BusinessImpact=BusinessImpactOption.MEDIUM, + Labels=[], + Owners=[], + ) + form_data = params.to_request_form() + + # Assert + assert form_data['Name'] == 'test-sbom' + assert form_data['Vendor'] == 'test-vendor' + assert form_data['BusinessImpact'] == BusinessImpactOption.MEDIUM + assert 'Labels[]' not in form_data + assert 'Owners[]' not in form_data + + +class TestSbomCommand: + """Tests for sbom_command with CLI integration""" + + @responses.activate + def test_sbom_command_happy_path( + self, + mocker: 'MockerFixture', + api_token_response: responses.Response, + import_sbom_client: ImportSbomClient, + ) -> None: + responses.add(api_token_response) + mocked_import_sbom.mock_import_sbom_responses(responses, import_sbom_client) + + with tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) as temp_file: + temp_file.write('{"sbom": "content"}') + temp_file_path = temp_file.name + + try: + args = [ + 'import', + 'sbom', + '--name', + 'test-sbom', + '--vendor', + 'test-vendor', + temp_file_path, + ] + result = CliRunner().invoke(app, args, env=CLI_ENV_VARS) + + assert result.exit_code == 0 + _validate_called_endpoint(responses.calls, ImportSbomClient.IMPORT_SBOM_REQUEST_PATH) + + finally: + Path(temp_file_path).unlink(missing_ok=True) + + @responses.activate + def test_sbom_command_with_all_options( + self, + mocker: 'MockerFixture', + api_token_response: responses.Response, + import_sbom_client: ImportSbomClient, + ) -> None: + responses.add(api_token_response) + mocked_import_sbom.mock_member_details_response(responses, import_sbom_client, 'user1@example.com', 'user-123') + mocked_import_sbom.mock_import_sbom_responses(responses, import_sbom_client) + + with tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) as temp_file: + temp_file.write('{"sbom": "content"}') + temp_file_path = temp_file.name + + try: + args = [ + 'import', + 'sbom', + '--name', + 'test-sbom', + '--vendor', + 'test-vendor', + '--label', + 'production', + '--label', + 'critical', + '--owner', + 'user1@example.com', + '--business-impact', + 'High', + temp_file_path, + ] + result = CliRunner().invoke(app, args, env=CLI_ENV_VARS) + + # Assert + assert result.exit_code == 0 + _validate_called_endpoint(responses.calls, ImportSbomClient.IMPORT_SBOM_REQUEST_PATH) + _validate_called_endpoint(responses.calls, ImportSbomClient.GET_USER_ID_REQUEST_PATH) + + finally: + Path(temp_file_path).unlink(missing_ok=True) + + def test_sbom_command_file_not_exists(self, mocker: 'MockerFixture') -> None: + from uuid import uuid4 + + non_existent_file = str(uuid4()) + + args = [ + 'import', + 'sbom', + '--name', + 'test-sbom', + '--vendor', + 'test-vendor', + non_existent_file, + ] + result = CliRunner().invoke(app, args, env=CLI_ENV_VARS) + + assert result.exit_code != 0 + assert "Invalid value for 'PATH': File " in result.output + assert 'not exist' in result.output + + def test_sbom_command_file_is_directory(self, mocker: 'MockerFixture') -> None: + with tempfile.TemporaryDirectory() as temp_dir: + args = [ + 'import', + 'sbom', + '--name', + 'test-sbom', + '--vendor', + 'test-vendor', + temp_dir, + ] + result = CliRunner().invoke(app, args, env=CLI_ENV_VARS) + + # Typer should reject this before the command runs + assert result.exit_code != 0 + # The error message contains "is a" and "directory" (may be across lines) + assert 'directory' in result.output.lower() + + @responses.activate + def test_sbom_command_invalid_owner_email( + self, + mocker: 'MockerFixture', + api_token_response: responses.Response, + import_sbom_client: ImportSbomClient, + ) -> None: + responses.add(api_token_response) + # Mock with no external_id to simulate user not found + mocked_import_sbom.mock_member_details_response(responses, import_sbom_client, 'invalid@example.com', None) + + with tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) as temp_file: + temp_file.write('{"sbom": "content"}') + temp_file_path = temp_file.name + + try: + args = [ + 'import', + 'sbom', + '--name', + 'test-sbom', + '--vendor', + 'test-vendor', + '--owner', + 'invalid@example.com', + temp_file_path, + ] + result = CliRunner().invoke(app, args, env=CLI_ENV_VARS) + + assert result.exit_code == 1 + assert 'invalid@example.com' in result.output + assert 'Failed to find user' in result.output or 'not found' in result.output.lower() + finally: + Path(temp_file_path).unlink(missing_ok=True) + + @responses.activate + def test_sbom_command_http_400_error( + self, + mocker: 'MockerFixture', + api_token_response: responses.Response, + import_sbom_client: ImportSbomClient, + ) -> None: + responses.add(api_token_response) + # Mock the SBOM import API endpoint to return 400 + mocked_import_sbom.mock_import_sbom_responses(responses, import_sbom_client, status=400) + + with tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) as temp_file: + temp_file.write('{"sbom": "content"}') + temp_file_path = temp_file.name + + try: + args = [ + 'import', + 'sbom', + '--name', + 'test-sbom', + '--vendor', + 'test-vendor', + temp_file_path, + ] + result = CliRunner().invoke(app, args, env=CLI_ENV_VARS) + + # HTTP 400 errors are also soft failures - exit with code 0 + assert result.exit_code == 0 + _validate_called_endpoint(responses.calls, ImportSbomClient.IMPORT_SBOM_REQUEST_PATH) + finally: + Path(temp_file_path).unlink(missing_ok=True) + + @responses.activate + def test_sbom_command_http_500_error( + self, + mocker: 'MockerFixture', + api_token_response: responses.Response, + import_sbom_client: ImportSbomClient, + ) -> None: + responses.add(api_token_response) + # Mock the SBOM import API endpoint to return 500 (soft failure) + mocked_import_sbom.mock_import_sbom_responses(responses, import_sbom_client, status=500) + + with tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) as temp_file: + temp_file.write('{"sbom": "content"}') + temp_file_path = temp_file.name + + try: + args = [ + 'import', + 'sbom', + '--name', + 'test-sbom', + '--vendor', + 'test-vendor', + temp_file_path, + ] + result = CliRunner().invoke(app, args, env=CLI_ENV_VARS) + + assert result.exit_code == 0 + _validate_called_endpoint(responses.calls, ImportSbomClient.IMPORT_SBOM_REQUEST_PATH) + finally: + Path(temp_file_path).unlink(missing_ok=True) + + @responses.activate + def test_sbom_command_multiple_owners( + self, + mocker: 'MockerFixture', + api_token_response: responses.Response, + import_sbom_client: ImportSbomClient, + ) -> None: + username1 = 'user1' + username2 = 'user2' + + responses.add(api_token_response) + mocked_import_sbom.mock_import_sbom_responses(responses, import_sbom_client) + mocked_import_sbom.mock_member_details_response( + responses, import_sbom_client, f'{username1}@example.com', 'user-123' + ) + mocked_import_sbom.mock_member_details_response( + responses, import_sbom_client, f'{username2}@example.com', 'user-456' + ) + + with tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) as temp_file: + temp_file.write('{"sbom": "content"}') + temp_file_path = temp_file.name + + try: + args = [ + 'import', + 'sbom', + '--name', + 'test-sbom', + '--vendor', + 'test-vendor', + '--owner', + f'{username1}@example.com', + '--owner', + f'{username2}@example.com', + temp_file_path, + ] + result = CliRunner().invoke(app, args, env=CLI_ENV_VARS) + + assert result.exit_code == 0 + _validate_called_endpoint(responses.calls, ImportSbomClient.IMPORT_SBOM_REQUEST_PATH) + _validate_called_endpoint(responses.calls, ImportSbomClient.GET_USER_ID_REQUEST_PATH, 2) + finally: + Path(temp_file_path).unlink(missing_ok=True) diff --git a/tests/cli/commands/scan/__init__.py b/tests/cli/commands/scan/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cli/commands/scan/test_code_scanner.py b/tests/cli/commands/scan/test_code_scanner.py new file mode 100644 index 00000000..8a9f60b7 --- /dev/null +++ b/tests/cli/commands/scan/test_code_scanner.py @@ -0,0 +1,164 @@ +import os +from os.path import normpath +from unittest.mock import MagicMock, Mock, patch + +from cycode.cli import consts +from cycode.cli.apps.scan.code_scanner import scan_disk_files +from cycode.cli.files_collector.file_excluder import _is_file_relevant_for_sca_scan +from cycode.cli.files_collector.path_documents import _generate_document +from cycode.cli.models import Document + + +def test_is_file_relevant_for_sca_scan() -> None: + path = os.path.join('some_package', 'node_modules', 'package.json') + assert _is_file_relevant_for_sca_scan(path) is False + path = os.path.join('some_package', 'node_modules', 'package.lock') + assert _is_file_relevant_for_sca_scan(path) is False + path = os.path.join('some_package', 'package.json') + assert _is_file_relevant_for_sca_scan(path) is True + path = os.path.join('some_package', 'package.lock') + assert _is_file_relevant_for_sca_scan(path) is True + + +def test_generate_document() -> None: + is_git_diff = False + + path = 'path/to/nowhere.txt' + content = 'nothing important here' + + non_iac_document = Document(path, content, is_git_diff) + generated_document = _generate_document(path, consts.SCA_SCAN_TYPE, content, is_git_diff) + + assert non_iac_document.path == generated_document.path + assert non_iac_document.content == generated_document.content + assert non_iac_document.is_git_diff_format == generated_document.is_git_diff_format + + path = 'path/to/nowhere.tf' + content = """provider "aws" { + profile = "chili" + region = "us-east-1" + } + + resource "aws_s3_bucket" "chili-env-var-test" { + bucket = "chili-env-var-test" + }""" + + iac_document = Document(path, content, is_git_diff) + generated_document = _generate_document(path, consts.IAC_SCAN_TYPE, content, is_git_diff) + assert iac_document.path == generated_document.path + assert iac_document.content == generated_document.content + assert iac_document.is_git_diff_format == generated_document.is_git_diff_format + + content = """ + { + "resource_changes":[ + { + "type":"aws_s3_bucket_public_access_block", + "name":"efrat-env-var-test", + "change":{ + "actions":[ + "create" + ], + "after":{ + "block_public_acls":false, + "block_public_policy":true, + "ignore_public_acls":false, + "restrict_public_buckets":true + } + } + ] + } + """ + + generated_tfplan_document = _generate_document(path, consts.IAC_SCAN_TYPE, content, is_git_diff) + + assert isinstance(generated_tfplan_document, Document) + assert generated_tfplan_document.path.endswith('.tf') + assert generated_tfplan_document.is_git_diff_format == is_git_diff + + +@patch('cycode.cli.apps.scan.code_scanner.get_relevant_documents') +@patch('cycode.cli.apps.scan.code_scanner.scan_documents') +@patch('cycode.cli.apps.scan.code_scanner.get_scan_parameters') +@patch('cycode.cli.apps.scan.code_scanner.os.path.isdir') +def test_entrypoint_cycode_added_to_documents( + mock_isdir: Mock, + mock_get_scan_parameters: Mock, + mock_scan_documents: Mock, + mock_get_relevant_documents: Mock, +) -> None: + """Test that entrypoint.cycode file is added to documents in scan_disk_files.""" + # Arrange + mock_ctx = MagicMock() + mock_ctx.obj = { + 'scan_type': consts.SAST_SCAN_TYPE, + 'progress_bar': MagicMock(), + } + mock_get_scan_parameters.return_value = {} + mock_isdir.return_value = True # Path is a directory + + mock_documents = [ + Document('/test/path/file1.py', 'content1', is_git_diff_format=False), + Document('/test/path/file2.js', 'content2', is_git_diff_format=False), + ] + mock_get_relevant_documents.return_value = mock_documents.copy() + test_path = '/Users/test/repositories' + + # Act + scan_disk_files(mock_ctx, (test_path,)) + + # Assert + call_args = mock_scan_documents.call_args + documents_passed = call_args[0][1] + + # Verify entrypoint document was added + entrypoint_docs = [doc for doc in documents_passed if doc.path.endswith(consts.CYCODE_ENTRYPOINT_FILENAME)] + assert len(entrypoint_docs) == 1 + + entrypoint_doc = entrypoint_docs[0] + # Normalize paths for cross-platform compatibility + expected_path = normpath(os.path.join(os.path.abspath(test_path), consts.CYCODE_ENTRYPOINT_FILENAME)) + assert normpath(entrypoint_doc.path) == expected_path + assert entrypoint_doc.content == '' + assert entrypoint_doc.is_git_diff_format is False + assert normpath(entrypoint_doc.absolute_path) == normpath(entrypoint_doc.path) + + +@patch('cycode.cli.apps.scan.code_scanner.get_relevant_documents') +@patch('cycode.cli.apps.scan.code_scanner.scan_documents') +@patch('cycode.cli.apps.scan.code_scanner.get_scan_parameters') +@patch('cycode.cli.apps.scan.code_scanner.os.path.isdir') +def test_entrypoint_cycode_not_added_for_single_file( + mock_isdir: Mock, + mock_get_scan_parameters: Mock, + mock_scan_documents: Mock, + mock_get_relevant_documents: Mock, +) -> None: + """Test that entrypoint.cycode file is NOT added when path is a single file.""" + # Arrange + mock_ctx = MagicMock() + mock_ctx.obj = { + 'scan_type': consts.SAST_SCAN_TYPE, + 'progress_bar': MagicMock(), + } + mock_get_scan_parameters.return_value = {} + mock_isdir.return_value = False # Path is a file, not a directory + + mock_documents = [ + Document('/test/path/file1.py', 'content1', is_git_diff_format=False), + ] + mock_get_relevant_documents.return_value = mock_documents.copy() + test_path = '/Users/test/file.py' + + # Act + scan_disk_files(mock_ctx, (test_path,)) + + # Assert + call_args = mock_scan_documents.call_args + documents_passed = call_args[0][1] + + # Verify entrypoint document was NOT added + entrypoint_docs = [doc for doc in documents_passed if doc.path.endswith(consts.CYCODE_ENTRYPOINT_FILENAME)] + assert len(entrypoint_docs) == 0 + # Verify only the original documents are present + assert len(documents_passed) == len(mock_documents) diff --git a/tests/cli/commands/scan/test_detection_excluder.py b/tests/cli/commands/scan/test_detection_excluder.py new file mode 100644 index 00000000..d35787ab --- /dev/null +++ b/tests/cli/commands/scan/test_detection_excluder.py @@ -0,0 +1,22 @@ +from cycode.cli.apps.scan.detection_excluder import _does_severity_match_severity_threshold + + +def test_does_severity_match_severity_threshold() -> None: + assert _does_severity_match_severity_threshold('INFO', 'LOW') is False + + assert _does_severity_match_severity_threshold('LOW', 'LOW') is True + assert _does_severity_match_severity_threshold('LOW', 'MEDIUM') is False + + assert _does_severity_match_severity_threshold('MEDIUM', 'LOW') is True + assert _does_severity_match_severity_threshold('MEDIUM', 'MEDIUM') is True + assert _does_severity_match_severity_threshold('MEDIUM', 'HIGH') is False + + assert _does_severity_match_severity_threshold('HIGH', 'MEDIUM') is True + assert _does_severity_match_severity_threshold('HIGH', 'HIGH') is True + assert _does_severity_match_severity_threshold('HIGH', 'CRITICAL') is False + + assert _does_severity_match_severity_threshold('CRITICAL', 'HIGH') is True + assert _does_severity_match_severity_threshold('CRITICAL', 'CRITICAL') is True + + assert _does_severity_match_severity_threshold('NON_EXISTENT', 'LOW') is True + assert _does_severity_match_severity_threshold('LOW', 'NON_EXISTENT') is True diff --git a/tests/cli/commands/scan/test_scan_parameters.py b/tests/cli/commands/scan/test_scan_parameters.py new file mode 100644 index 00000000..6933e9bc --- /dev/null +++ b/tests/cli/commands/scan/test_scan_parameters.py @@ -0,0 +1,115 @@ +from unittest.mock import MagicMock, patch + +import pytest + +from cycode.cli.apps.scan.scan_parameters import _get_default_scan_parameters, get_scan_parameters + + +@pytest.fixture +def mock_context() -> MagicMock: + """Create a mock typer.Context for testing.""" + ctx = MagicMock() + ctx.obj = { + 'monitor': False, + 'report': False, + 'package-vulnerabilities': True, + 'license-compliance': True, + } + ctx.info_name = 'test-command' + return ctx + + +def test_get_default_scan_parameters(mock_context: MagicMock) -> None: + """Test that default scan parameters are correctly extracted from context.""" + params = _get_default_scan_parameters(mock_context) + + assert params['monitor'] is False + assert params['report'] is False + assert params['package_vulnerabilities'] is True + assert params['license_compliance'] is True + assert params['command_type'] == 'test_command' # hyphens replaced with underscores + assert 'aggregation_id' in params + + +def test_get_scan_parameters_without_paths(mock_context: MagicMock) -> None: + """Test get_scan_parameters returns only default params when no paths provided.""" + params = get_scan_parameters(mock_context) + + assert 'paths' not in params + assert 'remote_url' not in params + assert 'branch' not in params + assert params['monitor'] is False + + +@patch('cycode.cli.apps.scan.scan_parameters.get_remote_url_scan_parameter') +def test_get_scan_parameters_with_paths(mock_get_remote_url: MagicMock, mock_context: MagicMock) -> None: + """Test get_scan_parameters includes paths and remote_url when paths provided.""" + mock_get_remote_url.return_value = 'https://github.com/example/repo.git' + paths = ('/path/to/repo',) + + params = get_scan_parameters(mock_context, paths) + + assert params['paths'] == paths + assert params['remote_url'] == 'https://github.com/example/repo.git' + assert mock_context.obj['remote_url'] == 'https://github.com/example/repo.git' + + +@patch('cycode.cli.apps.scan.scan_parameters.get_remote_url_scan_parameter') +def test_get_scan_parameters_includes_branch_when_set(mock_get_remote_url: MagicMock, mock_context: MagicMock) -> None: + """Test that branch is included in scan_parameters when set in context.""" + mock_get_remote_url.return_value = None + mock_context.obj['branch'] = 'feature-branch' + paths = ('/path/to/repo',) + + params = get_scan_parameters(mock_context, paths) + + assert params['branch'] == 'feature-branch' + + +@patch('cycode.cli.apps.scan.scan_parameters.get_remote_url_scan_parameter') +def test_get_scan_parameters_excludes_branch_when_not_set( + mock_get_remote_url: MagicMock, mock_context: MagicMock +) -> None: + """Test that branch is not included in scan_parameters when not set in context.""" + mock_get_remote_url.return_value = None + # Ensure branch is not in context + mock_context.obj.pop('branch', None) + paths = ('/path/to/repo',) + + params = get_scan_parameters(mock_context, paths) + + assert 'branch' not in params + + +@patch('cycode.cli.apps.scan.scan_parameters.get_remote_url_scan_parameter') +def test_get_scan_parameters_excludes_branch_when_none(mock_get_remote_url: MagicMock, mock_context: MagicMock) -> None: + """Test that branch is not included when explicitly set to None.""" + mock_get_remote_url.return_value = None + mock_context.obj['branch'] = None + paths = ('/path/to/repo',) + + params = get_scan_parameters(mock_context, paths) + + assert 'branch' not in params + + +@patch('cycode.cli.apps.scan.scan_parameters.get_remote_url_scan_parameter') +def test_get_scan_parameters_branch_with_various_names(mock_get_remote_url: MagicMock, mock_context: MagicMock) -> None: + """Test branch parameter works with various branch naming conventions.""" + mock_get_remote_url.return_value = None + paths = ('/path/to/repo',) + + # Test main branch + mock_context.obj['branch'] = 'main' + params = get_scan_parameters(mock_context, paths) + assert params['branch'] == 'main' + + # Test feature branch with slashes + mock_context.obj['branch'] = 'feature/add-new-functionality' + params = get_scan_parameters(mock_context, paths) + assert params['branch'] == 'feature/add-new-functionality' + + # Test branch with special characters + mock_context.obj['branch'] = 'release-v1.0.0' + params = get_scan_parameters(mock_context, paths) + assert params['branch'] == 'release-v1.0.0' diff --git a/tests/cli/commands/scan/test_scan_result.py b/tests/cli/commands/scan/test_scan_result.py new file mode 100644 index 00000000..e85ca116 --- /dev/null +++ b/tests/cli/commands/scan/test_scan_result.py @@ -0,0 +1,47 @@ +import os + +from cycode.cli.apps.scan.scan_result import _get_file_name_from_detection +from cycode.cli.consts import IAC_SCAN_TYPE, SAST_SCAN_TYPE, SCA_SCAN_TYPE, SECRET_SCAN_TYPE + + +def test_get_file_name_from_detection_sca_uses_file_path() -> None: + raw_detection = { + 'detection_details': { + 'file_name': 'package.json', + 'file_path': '/repo/path/package.json', + }, + } + result = _get_file_name_from_detection(SCA_SCAN_TYPE, raw_detection) + assert result == '/repo/path/package.json' + + +def test_get_file_name_from_detection_iac_uses_file_path() -> None: + raw_detection = { + 'detection_details': { + 'file_name': 'main.tf', + 'file_path': '/repo/infra/main.tf', + }, + } + result = _get_file_name_from_detection(IAC_SCAN_TYPE, raw_detection) + assert result == '/repo/infra/main.tf' + + +def test_get_file_name_from_detection_sast_uses_file_path() -> None: + raw_detection = { + 'detection_details': { + 'file_path': '/repo/src/app.py', + }, + } + result = _get_file_name_from_detection(SAST_SCAN_TYPE, raw_detection) + assert result == '/repo/src/app.py' + + +def test_get_file_name_from_detection_secret_uses_file_path_and_file_name() -> None: + raw_detection = { + 'detection_details': { + 'file_path': '/repo/src', + 'file_name': '.env', + }, + } + result = _get_file_name_from_detection(SECRET_SCAN_TYPE, raw_detection) + assert result == os.path.join('/repo/src', '.env') diff --git a/tests/cli/commands/test_check_latest_version_on_close.py b/tests/cli/commands/test_check_latest_version_on_close.py new file mode 100644 index 00000000..eccadf93 --- /dev/null +++ b/tests/cli/commands/test_check_latest_version_on_close.py @@ -0,0 +1,72 @@ +from unittest.mock import patch + +import pytest +from typer.testing import CliRunner + +from cycode import __version__ +from cycode.cli.app import app +from cycode.cli.cli_types import OutputTypeOption +from cycode.cli.utils.version_checker import VersionChecker +from tests.conftest import CLI_ENV_VARS + +_NEW_LATEST_VERSION = '999.0.0' # Simulate a newer version available +_UPDATE_MESSAGE_PART = 'new release of cycode cli is available' + + +@patch.object(VersionChecker, 'check_for_update') +def test_version_check_with_json_output(mock_check_update: patch) -> None: + # When output is JSON, version check should be skipped + mock_check_update.return_value = _NEW_LATEST_VERSION + + args = ['--output', OutputTypeOption.JSON, 'version'] + result = CliRunner().invoke(app, args, env=CLI_ENV_VARS) + + # Version check message should not be present in JSON output + assert _UPDATE_MESSAGE_PART not in result.output.lower() + mock_check_update.assert_not_called() + + +@pytest.fixture +def mock_auth_info() -> 'patch': + # Mock the authorization info to avoid API calls + with patch('cycode.cli.apps.auth.auth_common.get_authorization_info', return_value=None) as mock: + yield mock + + +@pytest.mark.parametrize('command', ['version', 'status']) +@patch.object(VersionChecker, 'check_for_update') +def test_version_check_for_special_commands(mock_check_update: patch, mock_auth_info: patch, command: str) -> None: + # Version and status commands should always check the version without cache + mock_check_update.return_value = _NEW_LATEST_VERSION + + result = CliRunner().invoke(app, [command], env=CLI_ENV_VARS) + + # Version information should be present in output + assert _UPDATE_MESSAGE_PART in result.output.lower() + # Version check must be called without a cache + mock_check_update.assert_called_once_with(__version__, False) + + +@patch.object(VersionChecker, 'check_for_update') +def test_version_check_with_text_output(mock_check_update: patch) -> None: + # Regular commands with text output should check the version using cache + mock_check_update.return_value = _NEW_LATEST_VERSION + + args = ['version'] + result = CliRunner().invoke(app, args, env=CLI_ENV_VARS) + + # Version check message should be present in JSON output + assert _UPDATE_MESSAGE_PART in result.output.lower() + + +@patch.object(VersionChecker, 'check_for_update') +def test_version_check_disabled(mock_check_update: patch) -> None: + # When --no-update-notifier is used, version check should be skipped + mock_check_update.return_value = _NEW_LATEST_VERSION + + args = ['--no-update-notifier', 'version'] + result = CliRunner().invoke(app, args, env=CLI_ENV_VARS) + + # Version check message should not be present + assert _UPDATE_MESSAGE_PART not in result.output.lower() + mock_check_update.assert_not_called() diff --git a/tests/cli/commands/test_main_command.py b/tests/cli/commands/test_main_command.py new file mode 100644 index 00000000..2436adee --- /dev/null +++ b/tests/cli/commands/test_main_command.py @@ -0,0 +1,87 @@ +import json +from typing import TYPE_CHECKING +from uuid import uuid4 + +import pytest +import responses +from typer.testing import CliRunner + +from cycode.cli import consts +from cycode.cli.app import app +from cycode.cli.cli_types import OutputTypeOption, ScanTypeOption +from cycode.cli.utils.git_proxy import git_proxy +from tests.conftest import CLI_ENV_VARS, TEST_FILES_PATH, ZIP_CONTENT_PATH +from tests.cyclient.mocked_responses.scan_client import mock_remote_config_responses, mock_scan_async_responses + +_PATH_TO_SCAN = TEST_FILES_PATH.joinpath('zip_content').absolute() + +if TYPE_CHECKING: + from cycode.cyclient.scan_client import ScanClient + + +def _is_json(plain: str) -> bool: + try: + json.loads(plain) + return True + except (ValueError, TypeError): + return False + + +@responses.activate +@pytest.mark.parametrize('output', [OutputTypeOption.TEXT, OutputTypeOption.JSON]) +def test_passing_output_option(output: str, scan_client: 'ScanClient', api_token_response: responses.Response) -> None: + scan_type = consts.SECRET_SCAN_TYPE + scan_id = uuid4() + + mock_scan_async_responses(responses, scan_type, scan_client, scan_id, ZIP_CONTENT_PATH) + responses.add(api_token_response) + + args = ['--output', output, 'scan', '--soft-fail', 'path', str(_PATH_TO_SCAN)] + env = {'PYTEST_TEST_UNIQUE_ID': str(scan_id), **CLI_ENV_VARS} + result = CliRunner().invoke(app, args, env=env) + + except_json = output == 'json' + + assert _is_json(result.output) == except_json + + if except_json: + output = json.loads(result.output) + assert 'scan_ids' in output + else: + assert 'violation:' in result.output + + +@responses.activate +def test_optional_git_with_path_scan(scan_client: 'ScanClient', api_token_response: responses.Response) -> None: + mock_scan_async_responses(responses, consts.SECRET_SCAN_TYPE, scan_client, uuid4(), ZIP_CONTENT_PATH) + responses.add(api_token_response) + + # fake env without Git executable + git_proxy._set_dummy_git_proxy() + + args = ['--output', 'json', 'scan', 'path', str(_PATH_TO_SCAN)] + result = CliRunner().invoke(app, args, env=CLI_ENV_VARS) + + # do NOT expect error about not found Git executable + assert 'GIT_PYTHON_GIT_EXECUTABLE' not in result.output + + # reset the git proxy + git_proxy._set_git_proxy() + + +@responses.activate +def test_required_git_with_path_repository(scan_client: 'ScanClient', api_token_response: responses.Response) -> None: + mock_remote_config_responses(responses, ScanTypeOption.SECRET, scan_client) + responses.add(api_token_response) + + # fake env without Git executable + git_proxy._set_dummy_git_proxy() + + args = ['--output', OutputTypeOption.JSON, 'scan', 'repository', str(_PATH_TO_SCAN)] + result = CliRunner().invoke(app, args, env=CLI_ENV_VARS) + + # expect error about not found Git executable + assert 'GIT_PYTHON_GIT_EXECUTABLE' in result.output + + # reset the git proxy + git_proxy._set_git_proxy() diff --git a/tests/cli/commands/version/__init__.py b/tests/cli/commands/version/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cli/commands/version/test_version_checker.py b/tests/cli/commands/version/test_version_checker.py new file mode 100644 index 00000000..14d6150e --- /dev/null +++ b/tests/cli/commands/version/test_version_checker.py @@ -0,0 +1,129 @@ +import time +from typing import TYPE_CHECKING, Optional +from unittest.mock import MagicMock, patch + +import pytest + +if TYPE_CHECKING: + from pathlib import Path + +from cycode.cli.utils.version_checker import VersionChecker + + +@pytest.fixture +def version_checker() -> 'VersionChecker': + return VersionChecker() + + +@pytest.fixture +def version_checker_cached(tmp_path: 'Path', version_checker: 'VersionChecker') -> 'VersionChecker': + version_checker.cache_file = tmp_path / '.version_check' + return version_checker + + +class TestVersionChecker: + def test_parse_version_stable(self, version_checker: 'VersionChecker') -> None: + version = '1.2.3' + parts, is_pre = version_checker._parse_version(version) + + assert parts == [1, 2, 3] + assert not is_pre + + def test_parse_version_prerelease(self, version_checker: 'VersionChecker') -> None: + version = '1.2.3dev4' + parts, is_pre = version_checker._parse_version(version) + + assert parts == [1, 2, 3, 4] + assert is_pre + + def test_parse_version_complex(self, version_checker: 'VersionChecker') -> None: + version = '1.2.3.dev4.post5' + parts, is_pre = version_checker._parse_version(version) + + assert parts == [1, 2, 3, 4, 5] + assert is_pre + + def test_should_check_update_no_cache(self, version_checker_cached: 'VersionChecker') -> None: + assert version_checker_cached._should_check_update(is_prerelease=False) is True + + def test_should_check_update_invalid_cache(self, version_checker_cached: 'VersionChecker') -> None: + version_checker_cached.cache_file.write_text('invalid') + assert version_checker_cached._should_check_update(is_prerelease=False) is True + + def test_should_check_update_expired(self, version_checker_cached: 'VersionChecker') -> None: + # Write a timestamp from 8 days ago + old_time = time.time() - (8 * 24 * 60 * 60) + version_checker_cached.cache_file.write_text(str(old_time)) + + assert version_checker_cached._should_check_update(is_prerelease=False) is True + + def test_should_check_update_not_expired(self, version_checker_cached: 'VersionChecker') -> None: + # Write a recent timestamp + version_checker_cached.cache_file.write_text(str(time.time())) + + assert version_checker_cached._should_check_update(is_prerelease=False) is False + + def test_should_check_update_prerelease_daily(self, version_checker_cached: 'VersionChecker') -> None: + # Write a timestamp from 25 hours ago + old_time = time.time() - (25 * 60 * 60) + version_checker_cached.cache_file.write_text(str(old_time)) + + assert version_checker_cached._should_check_update(is_prerelease=True) is True + + @pytest.mark.parametrize( + ('current_version', 'latest_version', 'expected_result'), + [ + # Stable version comparisons + ('1.2.3', '1.2.4', '1.2.4'), # Higher patch version + ('1.2.3', '1.3.0', '1.3.0'), # Higher minor version + ('1.2.3', '2.0.0', '2.0.0'), # Higher major version + ('1.2.3', '1.2.3', None), # Same version + ('1.2.4', '1.2.3', None), # Current higher than latest + # Pre-release version comparisons + ('1.2.3dev1', '1.2.3', '1.2.3'), # Pre-release to stable + ('1.2.3', '1.2.4dev1', None), # Stable to pre-release + ('1.2.3dev1', '1.2.3dev2', '1.2.3dev2'), # Pre-release to higher pre-release + ('1.2.3dev2', '1.2.3dev1', None), # Pre-release to lower pre-release + # Edge cases + ('1.0.0dev1', '1.0.0', '1.0.0'), # Pre-release to same version stable + ('2.0.0', '2.0.0dev1', None), # Stable to same version pre-release + ('2.2.1.dev4', '2.2.0', None), # Pre-release to lower stable + ], + ) + def test_check_for_update_scenarios( + self, + version_checker_cached: 'VersionChecker', + current_version: str, + latest_version: str, + expected_result: Optional[str], + ) -> None: + with patch.multiple( + version_checker_cached, + _should_check_update=MagicMock(return_value=True), + get_latest_version=MagicMock(return_value=latest_version), + _update_last_check=MagicMock(), + ): + result = version_checker_cached.check_for_update(current_version) + assert result == expected_result + + def test_get_latest_version_success(self, version_checker: 'VersionChecker') -> None: + mock_response = MagicMock() + mock_response.json.return_value = {'info': {'version': '1.2.3'}} + with patch.object(version_checker, 'get', return_value=mock_response): + assert version_checker.get_latest_version() == '1.2.3' + + def test_get_latest_version_failure(self, version_checker: 'VersionChecker') -> None: + with patch.object(version_checker, 'get', side_effect=Exception): + assert version_checker.get_latest_version() is None + + def test_update_last_check(self, version_checker_cached: 'VersionChecker') -> None: + version_checker_cached._update_last_check() + assert version_checker_cached.cache_file.exists() + + timestamp = float(version_checker_cached.cache_file.read_text().strip()) + assert abs(timestamp - time.time()) < 1 # Should be within 1 second + + def test_update_last_check_permission_error(self, version_checker_cached: 'VersionChecker') -> None: + with patch('builtins.open', side_effect=IOError): + version_checker_cached._update_last_check() + # Should not raise an exception diff --git a/tests/cli/exceptions/__init__.py b/tests/cli/exceptions/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cli/exceptions/test_handle_scan_errors.py b/tests/cli/exceptions/test_handle_scan_errors.py new file mode 100644 index 00000000..ce72e9de --- /dev/null +++ b/tests/cli/exceptions/test_handle_scan_errors.py @@ -0,0 +1,80 @@ +from typing import TYPE_CHECKING, Any + +import click +import pytest +import typer +from requests import Response +from rich.traceback import Traceback + +from cycode.cli.cli_types import OutputTypeOption +from cycode.cli.console import console_err +from cycode.cli.exceptions import custom_exceptions +from cycode.cli.exceptions.handle_scan_errors import handle_scan_exception +from cycode.cli.printers import ConsolePrinter +from cycode.cli.utils.git_proxy import git_proxy + +if TYPE_CHECKING: + from _pytest.monkeypatch import MonkeyPatch + + +@pytest.fixture +def ctx() -> typer.Context: + ctx = typer.Context(click.Command('path'), obj={'verbose': False, 'output': OutputTypeOption.TEXT}) + ctx.obj['console_printer'] = ConsolePrinter(ctx) + return ctx + + +@pytest.mark.parametrize( + ('exception', 'expected_soft_fail'), + [ + (custom_exceptions.RequestHttpError(400, 'msg', Response()), True), + (custom_exceptions.ScanAsyncError('msg'), True), + (custom_exceptions.HttpUnauthorizedError('msg', Response()), True), + (custom_exceptions.ZipTooLargeError(1000), True), + (custom_exceptions.TfplanKeyError('msg'), True), + (git_proxy.get_invalid_git_repository_error()(), None), + ], +) +def test_handle_exception_soft_fail( + ctx: typer.Context, exception: custom_exceptions.CycodeError, expected_soft_fail: bool +) -> None: + with ctx: + handle_scan_exception(ctx, exception) + + assert ctx.obj.get('did_fail') is True + assert ctx.obj.get('soft_fail') is expected_soft_fail + + +def test_handle_exception_unhandled_error(ctx: typer.Context) -> None: + with ctx, pytest.raises(typer.Exit): + handle_scan_exception(ctx, ValueError('test')) + + assert ctx.obj.get('did_fail') is True + assert ctx.obj.get('soft_fail') is None + + +def test_handle_exception_click_error(ctx: typer.Context) -> None: + with ctx, pytest.raises(click.ClickException): + handle_scan_exception(ctx, click.ClickException('test')) + + assert ctx.obj.get('did_fail') is True + assert ctx.obj.get('soft_fail') is None + + +def test_handle_exception_verbose(monkeypatch: 'MonkeyPatch') -> None: + ctx = typer.Context(click.Command('path'), obj={'verbose': True, 'output': OutputTypeOption.TEXT}) + ctx.obj['console_printer'] = ConsolePrinter(ctx) + + error_text = 'test' + + def mock_console_print(obj: Any, *_, **__) -> None: + if isinstance(obj, str): + assert 'Correlation ID:' in obj + else: + assert isinstance(obj, Traceback) + assert error_text in str(obj.trace) + + monkeypatch.setattr(console_err, 'print', mock_console_print) + + with pytest.raises(typer.Exit): + handle_scan_exception(ctx, ValueError(error_text)) diff --git a/tests/cli/files_collector/__init__.py b/tests/cli/files_collector/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cli/files_collector/iac/__init__.py b/tests/cli/files_collector/iac/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cli/files_collector/iac/test_tf_content_generator.py b/tests/cli/files_collector/iac/test_tf_content_generator.py new file mode 100644 index 00000000..369fc936 --- /dev/null +++ b/tests/cli/files_collector/iac/test_tf_content_generator.py @@ -0,0 +1,17 @@ +import os + +from cycode.cli.files_collector.iac import tf_content_generator +from cycode.cli.utils.path_utils import get_file_content, get_immediate_subdirectories +from tests.conftest import TEST_FILES_PATH + +_PATH_TO_EXAMPLES = os.path.join(TEST_FILES_PATH, 'tf_content_generator_files') + + +def test_generate_tf_content_from_tfplan() -> None: + examples_directories = get_immediate_subdirectories(_PATH_TO_EXAMPLES) + for example in examples_directories: + tfplan_content = get_file_content(os.path.join(_PATH_TO_EXAMPLES, example, 'tfplan.json')) + tf_expected_content = get_file_content(os.path.join(_PATH_TO_EXAMPLES, example, 'tf_content.txt')) + tf_content = tf_content_generator.generate_tf_content_from_tfplan(example, tfplan_content) + + assert tf_content == tf_expected_content diff --git a/tests/cli/files_collector/sca/npm/__init__.py b/tests/cli/files_collector/sca/npm/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cli/files_collector/sca/npm/test_restore_deno_dependencies.py b/tests/cli/files_collector/sca/npm/test_restore_deno_dependencies.py new file mode 100644 index 00000000..2d6e9a4b --- /dev/null +++ b/tests/cli/files_collector/sca/npm/test_restore_deno_dependencies.py @@ -0,0 +1,65 @@ +from pathlib import Path +from unittest.mock import MagicMock + +import pytest +import typer + +from cycode.cli.files_collector.sca.npm.restore_deno_dependencies import ( + DENO_LOCK_FILE_NAME, + DENO_MANIFEST_FILE_NAMES, + RestoreDenoDependencies, +) +from cycode.cli.models import Document + + +@pytest.fixture +def mock_ctx(tmp_path: Path) -> typer.Context: + ctx = MagicMock(spec=typer.Context) + ctx.obj = {'monitor': False} + ctx.params = {'path': str(tmp_path)} + return ctx + + +@pytest.fixture +def restore_deno(mock_ctx: typer.Context) -> RestoreDenoDependencies: + return RestoreDenoDependencies(mock_ctx, is_git_diff=False, command_timeout=30) + + +class TestIsProject: + @pytest.mark.parametrize('filename', DENO_MANIFEST_FILE_NAMES) + def test_deno_manifest_files_match(self, restore_deno: RestoreDenoDependencies, filename: str) -> None: + doc = Document(filename, '{}') + assert restore_deno.is_project(doc) is True + + @pytest.mark.parametrize('filename', ['package.json', 'tsconfig.json', 'deno.ts', 'main.ts', 'deno.lock']) + def test_non_deno_manifest_files_do_not_match(self, restore_deno: RestoreDenoDependencies, filename: str) -> None: + doc = Document(filename, '') + assert restore_deno.is_project(doc) is False + + +class TestTryRestoreDependencies: + def test_existing_deno_lock_returned(self, restore_deno: RestoreDenoDependencies, tmp_path: Path) -> None: + deno_lock_content = '{"version": "3", "packages": {}}' + (tmp_path / 'deno.json').write_text('{"imports": {}}') + (tmp_path / 'deno.lock').write_text(deno_lock_content) + + doc = Document(str(tmp_path / 'deno.json'), '{"imports": {}}', absolute_path=str(tmp_path / 'deno.json')) + result = restore_deno.try_restore_dependencies(doc) + + assert result is not None + assert DENO_LOCK_FILE_NAME in result.path + assert result.content == deno_lock_content + + def test_no_deno_lock_returns_none(self, restore_deno: RestoreDenoDependencies, tmp_path: Path) -> None: + (tmp_path / 'deno.json').write_text('{"imports": {}}') + + doc = Document(str(tmp_path / 'deno.json'), '{"imports": {}}', absolute_path=str(tmp_path / 'deno.json')) + result = restore_deno.try_restore_dependencies(doc) + + assert result is None + + def test_get_lock_file_name(self, restore_deno: RestoreDenoDependencies) -> None: + assert restore_deno.get_lock_file_name() == DENO_LOCK_FILE_NAME + + def test_get_commands_returns_empty(self, restore_deno: RestoreDenoDependencies) -> None: + assert restore_deno.get_commands('/path/to/deno.json') == [] diff --git a/tests/cli/files_collector/sca/npm/test_restore_npm_dependencies.py b/tests/cli/files_collector/sca/npm/test_restore_npm_dependencies.py new file mode 100644 index 00000000..aa145de3 --- /dev/null +++ b/tests/cli/files_collector/sca/npm/test_restore_npm_dependencies.py @@ -0,0 +1,113 @@ +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest +import typer + +from cycode.cli.files_collector.sca.npm.restore_npm_dependencies import ( + NPM_LOCK_FILE_NAME, + RestoreNpmDependencies, +) +from cycode.cli.models import Document + + +@pytest.fixture +def mock_ctx(tmp_path: Path) -> typer.Context: + ctx = MagicMock(spec=typer.Context) + ctx.obj = {'monitor': False} + ctx.params = {'path': str(tmp_path)} + return ctx + + +@pytest.fixture +def restore_npm(mock_ctx: typer.Context) -> RestoreNpmDependencies: + return RestoreNpmDependencies(mock_ctx, is_git_diff=False, command_timeout=30) + + +class TestIsProject: + def test_package_json_with_no_lockfile_matches(self, restore_npm: RestoreNpmDependencies, tmp_path: Path) -> None: + (tmp_path / 'package.json').write_text('{"name": "test"}') + doc = Document(str(tmp_path / 'package.json'), '{"name": "test"}', absolute_path=str(tmp_path / 'package.json')) + assert restore_npm.is_project(doc) is True + + def test_package_json_with_yarn_lock_does_not_match( + self, restore_npm: RestoreNpmDependencies, tmp_path: Path + ) -> None: + """Yarn projects are handled by RestoreYarnDependencies — NPM should not claim them.""" + (tmp_path / 'package.json').write_text('{"name": "test"}') + (tmp_path / 'yarn.lock').write_text('# yarn lockfile v1\n') + doc = Document(str(tmp_path / 'package.json'), '{"name": "test"}', absolute_path=str(tmp_path / 'package.json')) + assert restore_npm.is_project(doc) is False + + def test_package_json_with_pnpm_lock_does_not_match( + self, restore_npm: RestoreNpmDependencies, tmp_path: Path + ) -> None: + """pnpm projects are handled by RestorePnpmDependencies — NPM should not claim them.""" + (tmp_path / 'package.json').write_text('{"name": "test"}') + (tmp_path / 'pnpm-lock.yaml').write_text('lockfileVersion: 5.4\n') + doc = Document(str(tmp_path / 'package.json'), '{"name": "test"}', absolute_path=str(tmp_path / 'package.json')) + assert restore_npm.is_project(doc) is False + + def test_tsconfig_json_does_not_match(self, restore_npm: RestoreNpmDependencies) -> None: + doc = Document('tsconfig.json', '{}') + assert restore_npm.is_project(doc) is False + + def test_arbitrary_json_does_not_match(self, restore_npm: RestoreNpmDependencies) -> None: + for filename in ('jest.config.json', '.eslintrc.json', 'settings.json', 'bom.json'): + doc = Document(filename, '{}') + assert restore_npm.is_project(doc) is False, f'Expected False for {filename}' + + def test_non_json_file_does_not_match(self, restore_npm: RestoreNpmDependencies) -> None: + for filename in ('readme.txt', 'script.js', 'Makefile'): + doc = Document(filename, '') + assert restore_npm.is_project(doc) is False, f'Expected False for {filename}' + + +class TestTryRestoreDependencies: + def test_no_lockfile_calls_base_class(self, restore_npm: RestoreNpmDependencies, tmp_path: Path) -> None: + """When no lockfile exists, the base class (npm install) should be invoked.""" + (tmp_path / 'package.json').write_text('{"name": "test"}') + doc = Document(str(tmp_path / 'package.json'), '{"name": "test"}', absolute_path=str(tmp_path / 'package.json')) + + with patch.object( + restore_npm.__class__.__bases__[0], 'try_restore_dependencies', return_value=None + ) as mock_super: + restore_npm.try_restore_dependencies(doc) + mock_super.assert_called_once_with(doc) + + def test_lockfile_in_different_directory_still_calls_base_class( + self, restore_npm: RestoreNpmDependencies, tmp_path: Path + ) -> None: + (tmp_path / 'package.json').write_text('{"name": "test"}') + other_dir = tmp_path / 'other' + other_dir.mkdir() + (other_dir / 'pnpm-lock.yaml').write_text('lockfileVersion: 5.4\n') + doc = Document(str(tmp_path / 'package.json'), '{"name": "test"}', absolute_path=str(tmp_path / 'package.json')) + + with patch.object( + restore_npm.__class__.__bases__[0], 'try_restore_dependencies', return_value=None + ) as mock_super: + restore_npm.try_restore_dependencies(doc) + mock_super.assert_called_once_with(doc) + + +class TestGetLockFileName: + def test_get_lock_file_name(self, restore_npm: RestoreNpmDependencies) -> None: + assert restore_npm.get_lock_file_name() == NPM_LOCK_FILE_NAME + + def test_get_lock_file_names_contains_only_npm_lock(self, restore_npm: RestoreNpmDependencies) -> None: + assert restore_npm.get_lock_file_names() == [NPM_LOCK_FILE_NAME] + + +class TestPrepareManifestFilePath: + def test_strips_package_json_filename(self, restore_npm: RestoreNpmDependencies) -> None: + path = str(Path('/path/to/package.json')) + expected = str(Path('/path/to')) + assert restore_npm.prepare_manifest_file_path_for_command(path) == expected + + def test_package_json_in_cwd_returns_empty_string(self, restore_npm: RestoreNpmDependencies) -> None: + assert restore_npm.prepare_manifest_file_path_for_command('package.json') == '' + + def test_non_package_json_path_returned_unchanged(self, restore_npm: RestoreNpmDependencies) -> None: + path = str(Path('/path/to/')) + assert restore_npm.prepare_manifest_file_path_for_command(path) == path diff --git a/tests/cli/files_collector/sca/npm/test_restore_pnpm_dependencies.py b/tests/cli/files_collector/sca/npm/test_restore_pnpm_dependencies.py new file mode 100644 index 00000000..312cce83 --- /dev/null +++ b/tests/cli/files_collector/sca/npm/test_restore_pnpm_dependencies.py @@ -0,0 +1,91 @@ +from pathlib import Path +from unittest.mock import MagicMock + +import pytest +import typer + +from cycode.cli.files_collector.sca.npm.restore_pnpm_dependencies import ( + PNPM_LOCK_FILE_NAME, + RestorePnpmDependencies, +) +from cycode.cli.models import Document + + +@pytest.fixture +def mock_ctx(tmp_path: Path) -> typer.Context: + ctx = MagicMock(spec=typer.Context) + ctx.obj = {'monitor': False} + ctx.params = {'path': str(tmp_path)} + return ctx + + +@pytest.fixture +def restore_pnpm(mock_ctx: typer.Context) -> RestorePnpmDependencies: + return RestorePnpmDependencies(mock_ctx, is_git_diff=False, command_timeout=30) + + +class TestIsProject: + def test_package_json_with_pnpm_lock_matches(self, restore_pnpm: RestorePnpmDependencies, tmp_path: Path) -> None: + (tmp_path / 'package.json').write_text('{"name": "test"}') + (tmp_path / 'pnpm-lock.yaml').write_text('lockfileVersion: 5.4\n') + doc = Document(str(tmp_path / 'package.json'), '{"name": "test"}', absolute_path=str(tmp_path / 'package.json')) + assert restore_pnpm.is_project(doc) is True + + def test_package_json_with_package_manager_pnpm_matches(self, restore_pnpm: RestorePnpmDependencies) -> None: + content = '{"name": "test", "packageManager": "pnpm@8.6.2"}' + doc = Document('package.json', content) + assert restore_pnpm.is_project(doc) is True + + def test_package_json_with_engines_pnpm_matches(self, restore_pnpm: RestorePnpmDependencies) -> None: + content = '{"name": "test", "engines": {"pnpm": ">=8"}}' + doc = Document('package.json', content) + assert restore_pnpm.is_project(doc) is True + + def test_package_json_with_no_pnpm_signal_does_not_match( + self, restore_pnpm: RestorePnpmDependencies, tmp_path: Path + ) -> None: + (tmp_path / 'package.json').write_text('{"name": "test"}') + doc = Document(str(tmp_path / 'package.json'), '{"name": "test"}', absolute_path=str(tmp_path / 'package.json')) + assert restore_pnpm.is_project(doc) is False + + def test_package_json_with_yarn_lock_does_not_match( + self, restore_pnpm: RestorePnpmDependencies, tmp_path: Path + ) -> None: + (tmp_path / 'package.json').write_text('{"name": "test"}') + (tmp_path / 'yarn.lock').write_text('# yarn lockfile v1\n') + doc = Document(str(tmp_path / 'package.json'), '{"name": "test"}', absolute_path=str(tmp_path / 'package.json')) + assert restore_pnpm.is_project(doc) is False + + def test_tsconfig_json_does_not_match(self, restore_pnpm: RestorePnpmDependencies) -> None: + doc = Document('tsconfig.json', '{"compilerOptions": {}}') + assert restore_pnpm.is_project(doc) is False + + def test_package_manager_yarn_does_not_match(self, restore_pnpm: RestorePnpmDependencies) -> None: + content = '{"name": "test", "packageManager": "yarn@4.0.0"}' + doc = Document('package.json', content) + assert restore_pnpm.is_project(doc) is False + + def test_invalid_json_content_does_not_match(self, restore_pnpm: RestorePnpmDependencies) -> None: + doc = Document('package.json', 'not valid json') + assert restore_pnpm.is_project(doc) is False + + +class TestTryRestoreDependencies: + def test_existing_pnpm_lock_returned_directly(self, restore_pnpm: RestorePnpmDependencies, tmp_path: Path) -> None: + pnpm_lock_content = 'lockfileVersion: 5.4\n\npackages:\n /package@1.0.0:\n resolution: {}\n' + (tmp_path / 'package.json').write_text('{"name": "test"}') + (tmp_path / 'pnpm-lock.yaml').write_text(pnpm_lock_content) + + doc = Document(str(tmp_path / 'package.json'), '{"name": "test"}', absolute_path=str(tmp_path / 'package.json')) + result = restore_pnpm.try_restore_dependencies(doc) + + assert result is not None + assert PNPM_LOCK_FILE_NAME in result.path + assert result.content == pnpm_lock_content + + def test_get_lock_file_name(self, restore_pnpm: RestorePnpmDependencies) -> None: + assert restore_pnpm.get_lock_file_name() == PNPM_LOCK_FILE_NAME + + def test_get_commands_returns_pnpm_install(self, restore_pnpm: RestorePnpmDependencies) -> None: + commands = restore_pnpm.get_commands('/path/to/package.json') + assert commands == [['pnpm', 'install', '--ignore-scripts']] diff --git a/tests/cli/files_collector/sca/npm/test_restore_yarn_dependencies.py b/tests/cli/files_collector/sca/npm/test_restore_yarn_dependencies.py new file mode 100644 index 00000000..13e321c9 --- /dev/null +++ b/tests/cli/files_collector/sca/npm/test_restore_yarn_dependencies.py @@ -0,0 +1,91 @@ +from pathlib import Path +from unittest.mock import MagicMock + +import pytest +import typer + +from cycode.cli.files_collector.sca.npm.restore_yarn_dependencies import ( + YARN_LOCK_FILE_NAME, + RestoreYarnDependencies, +) +from cycode.cli.models import Document + + +@pytest.fixture +def mock_ctx(tmp_path: Path) -> typer.Context: + ctx = MagicMock(spec=typer.Context) + ctx.obj = {'monitor': False} + ctx.params = {'path': str(tmp_path)} + return ctx + + +@pytest.fixture +def restore_yarn(mock_ctx: typer.Context) -> RestoreYarnDependencies: + return RestoreYarnDependencies(mock_ctx, is_git_diff=False, command_timeout=30) + + +class TestIsProject: + def test_package_json_with_yarn_lock_matches(self, restore_yarn: RestoreYarnDependencies, tmp_path: Path) -> None: + (tmp_path / 'package.json').write_text('{"name": "test"}') + (tmp_path / 'yarn.lock').write_text('# yarn lockfile v1\n') + doc = Document(str(tmp_path / 'package.json'), '{"name": "test"}', absolute_path=str(tmp_path / 'package.json')) + assert restore_yarn.is_project(doc) is True + + def test_package_json_with_package_manager_yarn_matches(self, restore_yarn: RestoreYarnDependencies) -> None: + content = '{"name": "test", "packageManager": "yarn@4.0.2"}' + doc = Document('package.json', content) + assert restore_yarn.is_project(doc) is True + + def test_package_json_with_engines_yarn_matches(self, restore_yarn: RestoreYarnDependencies) -> None: + content = '{"name": "test", "engines": {"yarn": ">=1.22"}}' + doc = Document('package.json', content) + assert restore_yarn.is_project(doc) is True + + def test_package_json_with_no_yarn_signal_does_not_match( + self, restore_yarn: RestoreYarnDependencies, tmp_path: Path + ) -> None: + (tmp_path / 'package.json').write_text('{"name": "test"}') + doc = Document(str(tmp_path / 'package.json'), '{"name": "test"}', absolute_path=str(tmp_path / 'package.json')) + assert restore_yarn.is_project(doc) is False + + def test_package_json_with_pnpm_lock_does_not_match( + self, restore_yarn: RestoreYarnDependencies, tmp_path: Path + ) -> None: + (tmp_path / 'package.json').write_text('{"name": "test"}') + (tmp_path / 'pnpm-lock.yaml').write_text('lockfileVersion: 5.4\n') + doc = Document(str(tmp_path / 'package.json'), '{"name": "test"}', absolute_path=str(tmp_path / 'package.json')) + assert restore_yarn.is_project(doc) is False + + def test_tsconfig_json_does_not_match(self, restore_yarn: RestoreYarnDependencies) -> None: + doc = Document('tsconfig.json', '{"compilerOptions": {}}') + assert restore_yarn.is_project(doc) is False + + def test_package_manager_npm_does_not_match(self, restore_yarn: RestoreYarnDependencies) -> None: + content = '{"name": "test", "packageManager": "npm@9.0.0"}' + doc = Document('package.json', content) + assert restore_yarn.is_project(doc) is False + + def test_invalid_json_content_does_not_match(self, restore_yarn: RestoreYarnDependencies) -> None: + doc = Document('package.json', 'not valid json') + assert restore_yarn.is_project(doc) is False + + +class TestTryRestoreDependencies: + def test_existing_yarn_lock_returned_directly(self, restore_yarn: RestoreYarnDependencies, tmp_path: Path) -> None: + yarn_lock_content = '# yarn lockfile v1\n\npackage@1.0.0:\n resolved "https://example.com"\n' + (tmp_path / 'package.json').write_text('{"name": "test"}') + (tmp_path / 'yarn.lock').write_text(yarn_lock_content) + + doc = Document(str(tmp_path / 'package.json'), '{"name": "test"}', absolute_path=str(tmp_path / 'package.json')) + result = restore_yarn.try_restore_dependencies(doc) + + assert result is not None + assert YARN_LOCK_FILE_NAME in result.path + assert result.content == yarn_lock_content + + def test_get_lock_file_name(self, restore_yarn: RestoreYarnDependencies) -> None: + assert restore_yarn.get_lock_file_name() == YARN_LOCK_FILE_NAME + + def test_get_commands_returns_yarn_install(self, restore_yarn: RestoreYarnDependencies) -> None: + commands = restore_yarn.get_commands('/path/to/package.json') + assert commands == [['yarn', 'install', '--ignore-scripts']] diff --git a/tests/cli/files_collector/sca/php/__init__.py b/tests/cli/files_collector/sca/php/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cli/files_collector/sca/php/test_restore_composer_dependencies.py b/tests/cli/files_collector/sca/php/test_restore_composer_dependencies.py new file mode 100644 index 00000000..463eeddb --- /dev/null +++ b/tests/cli/files_collector/sca/php/test_restore_composer_dependencies.py @@ -0,0 +1,82 @@ +from pathlib import Path +from unittest.mock import MagicMock + +import pytest +import typer + +from cycode.cli.files_collector.sca.php.restore_composer_dependencies import ( + COMPOSER_LOCK_FILE_NAME, + RestoreComposerDependencies, +) +from cycode.cli.models import Document + + +@pytest.fixture +def mock_ctx(tmp_path: Path) -> typer.Context: + ctx = MagicMock(spec=typer.Context) + ctx.obj = {'monitor': False} + ctx.params = {'path': str(tmp_path)} + return ctx + + +@pytest.fixture +def restore_composer(mock_ctx: typer.Context) -> RestoreComposerDependencies: + return RestoreComposerDependencies(mock_ctx, is_git_diff=False, command_timeout=30) + + +class TestIsProject: + def test_composer_json_matches(self, restore_composer: RestoreComposerDependencies) -> None: + doc = Document('composer.json', '{"name": "vendor/project"}\n') + assert restore_composer.is_project(doc) is True + + def test_composer_json_in_subdir_matches(self, restore_composer: RestoreComposerDependencies) -> None: + doc = Document('myapp/composer.json', '{"name": "vendor/project"}\n') + assert restore_composer.is_project(doc) is True + + def test_composer_lock_does_not_match(self, restore_composer: RestoreComposerDependencies) -> None: + doc = Document('composer.lock', '{"_readme": []}\n') + assert restore_composer.is_project(doc) is False + + def test_package_json_does_not_match(self, restore_composer: RestoreComposerDependencies) -> None: + doc = Document('package.json', '{"name": "test"}\n') + assert restore_composer.is_project(doc) is False + + def test_other_json_does_not_match(self, restore_composer: RestoreComposerDependencies) -> None: + doc = Document('config.json', '{"setting": "value"}\n') + assert restore_composer.is_project(doc) is False + + +class TestTryRestoreDependencies: + def test_existing_composer_lock_returned_directly( + self, restore_composer: RestoreComposerDependencies, tmp_path: Path + ) -> None: + lock_content = '{\n "_readme": ["This file is @generated by Composer"],\n "packages": []\n}\n' + (tmp_path / 'composer.json').write_text('{"name": "vendor/project"}\n') + (tmp_path / 'composer.lock').write_text(lock_content) + + doc = Document( + str(tmp_path / 'composer.json'), + '{"name": "vendor/project"}\n', + absolute_path=str(tmp_path / 'composer.json'), + ) + result = restore_composer.try_restore_dependencies(doc) + + assert result is not None + assert COMPOSER_LOCK_FILE_NAME in result.path + assert result.content == lock_content + + def test_get_lock_file_name(self, restore_composer: RestoreComposerDependencies) -> None: + assert restore_composer.get_lock_file_name() == COMPOSER_LOCK_FILE_NAME + + def test_get_commands_returns_composer_update(self, restore_composer: RestoreComposerDependencies) -> None: + commands = restore_composer.get_commands('/path/to/composer.json') + assert commands == [ + [ + 'composer', + 'update', + '--no-cache', + '--no-install', + '--no-scripts', + '--ignore-platform-reqs', + ] + ] diff --git a/tests/cli/files_collector/sca/python/__init__.py b/tests/cli/files_collector/sca/python/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cli/files_collector/sca/python/test_restore_pipenv_dependencies.py b/tests/cli/files_collector/sca/python/test_restore_pipenv_dependencies.py new file mode 100644 index 00000000..9d34a7e3 --- /dev/null +++ b/tests/cli/files_collector/sca/python/test_restore_pipenv_dependencies.py @@ -0,0 +1,73 @@ +from pathlib import Path +from unittest.mock import MagicMock + +import pytest +import typer + +from cycode.cli.files_collector.sca.python.restore_pipenv_dependencies import ( + PIPENV_LOCK_FILE_NAME, + RestorePipenvDependencies, +) +from cycode.cli.models import Document + + +@pytest.fixture +def mock_ctx(tmp_path: Path) -> typer.Context: + ctx = MagicMock(spec=typer.Context) + ctx.obj = {'monitor': False} + ctx.params = {'path': str(tmp_path)} + return ctx + + +@pytest.fixture +def restore_pipenv(mock_ctx: typer.Context) -> RestorePipenvDependencies: + return RestorePipenvDependencies(mock_ctx, is_git_diff=False, command_timeout=30) + + +class TestIsProject: + def test_pipfile_matches(self, restore_pipenv: RestorePipenvDependencies) -> None: + doc = Document('Pipfile', '[[source]]\nname = "pypi"\n') + assert restore_pipenv.is_project(doc) is True + + def test_pipfile_in_subdir_matches(self, restore_pipenv: RestorePipenvDependencies) -> None: + doc = Document('myapp/Pipfile', '[[source]]\nname = "pypi"\n') + assert restore_pipenv.is_project(doc) is True + + def test_pipfile_lock_does_not_match(self, restore_pipenv: RestorePipenvDependencies) -> None: + doc = Document('Pipfile.lock', '{"default": {}}\n') + assert restore_pipenv.is_project(doc) is False + + def test_requirements_txt_does_not_match(self, restore_pipenv: RestorePipenvDependencies) -> None: + doc = Document('requirements.txt', 'requests==2.31.0\n') + assert restore_pipenv.is_project(doc) is False + + def test_pyproject_toml_does_not_match(self, restore_pipenv: RestorePipenvDependencies) -> None: + doc = Document('pyproject.toml', '[build-system]\nrequires = ["setuptools"]\n') + assert restore_pipenv.is_project(doc) is False + + +class TestTryRestoreDependencies: + def test_existing_pipfile_lock_returned_directly( + self, restore_pipenv: RestorePipenvDependencies, tmp_path: Path + ) -> None: + lock_content = '{"_meta": {"hash": {"sha256": "abc"}}, "default": {}, "develop": {}}\n' + (tmp_path / 'Pipfile').write_text('[[source]]\nname = "pypi"\n') + (tmp_path / 'Pipfile.lock').write_text(lock_content) + + doc = Document( + str(tmp_path / 'Pipfile'), + '[[source]]\nname = "pypi"\n', + absolute_path=str(tmp_path / 'Pipfile'), + ) + result = restore_pipenv.try_restore_dependencies(doc) + + assert result is not None + assert PIPENV_LOCK_FILE_NAME in result.path + assert result.content == lock_content + + def test_get_lock_file_name(self, restore_pipenv: RestorePipenvDependencies) -> None: + assert restore_pipenv.get_lock_file_name() == PIPENV_LOCK_FILE_NAME + + def test_get_commands_returns_pipenv_lock(self, restore_pipenv: RestorePipenvDependencies) -> None: + commands = restore_pipenv.get_commands('/path/to/Pipfile') + assert commands == [['pipenv', 'lock']] diff --git a/tests/cli/files_collector/sca/python/test_restore_poetry_dependencies.py b/tests/cli/files_collector/sca/python/test_restore_poetry_dependencies.py new file mode 100644 index 00000000..73f0d14f --- /dev/null +++ b/tests/cli/files_collector/sca/python/test_restore_poetry_dependencies.py @@ -0,0 +1,99 @@ +from pathlib import Path +from unittest.mock import MagicMock + +import pytest +import typer + +from cycode.cli.files_collector.sca.python.restore_poetry_dependencies import ( + POETRY_LOCK_FILE_NAME, + RestorePoetryDependencies, +) +from cycode.cli.models import Document + + +@pytest.fixture +def mock_ctx(tmp_path: Path) -> typer.Context: + ctx = MagicMock(spec=typer.Context) + ctx.obj = {'monitor': False} + ctx.params = {'path': str(tmp_path)} + return ctx + + +@pytest.fixture +def restore_poetry(mock_ctx: typer.Context) -> RestorePoetryDependencies: + return RestorePoetryDependencies(mock_ctx, is_git_diff=False, command_timeout=30) + + +class TestIsProject: + def test_pyproject_toml_with_poetry_lock_matches( + self, restore_poetry: RestorePoetryDependencies, tmp_path: Path + ) -> None: + (tmp_path / 'pyproject.toml').write_text('[tool.poetry]\nname = "test"\n') + (tmp_path / 'poetry.lock').write_text('# This file is generated by Poetry\n') + doc = Document( + str(tmp_path / 'pyproject.toml'), + '[tool.poetry]\nname = "test"\n', + absolute_path=str(tmp_path / 'pyproject.toml'), + ) + assert restore_poetry.is_project(doc) is True + + def test_pyproject_toml_with_tool_poetry_section_matches(self, restore_poetry: RestorePoetryDependencies) -> None: + content = '[tool.poetry]\nname = "my-project"\nversion = "1.0.0"\n' + doc = Document('pyproject.toml', content) + assert restore_poetry.is_project(doc) is True + + def test_pyproject_toml_without_poetry_section_does_not_match( + self, restore_poetry: RestorePoetryDependencies, tmp_path: Path + ) -> None: + content = '[build-system]\nrequires = ["setuptools"]\n' + (tmp_path / 'pyproject.toml').write_text(content) + doc = Document( + str(tmp_path / 'pyproject.toml'), + content, + absolute_path=str(tmp_path / 'pyproject.toml'), + ) + assert restore_poetry.is_project(doc) is False + + def test_requirements_txt_does_not_match(self, restore_poetry: RestorePoetryDependencies) -> None: + doc = Document('requirements.txt', 'requests==2.31.0\n') + assert restore_poetry.is_project(doc) is False + + def test_setup_py_does_not_match(self, restore_poetry: RestorePoetryDependencies) -> None: + doc = Document('setup.py', 'from setuptools import setup\nsetup()\n') + assert restore_poetry.is_project(doc) is False + + def test_empty_content_does_not_match(self, restore_poetry: RestorePoetryDependencies, tmp_path: Path) -> None: + (tmp_path / 'pyproject.toml').write_text('') + doc = Document( + str(tmp_path / 'pyproject.toml'), + '', + absolute_path=str(tmp_path / 'pyproject.toml'), + ) + assert restore_poetry.is_project(doc) is False + + +class TestTryRestoreDependencies: + def test_existing_poetry_lock_returned_directly( + self, restore_poetry: RestorePoetryDependencies, tmp_path: Path + ) -> None: + lock_content = '# This file is generated by Poetry\n\n[[package]]\nname = "requests"\n' + (tmp_path / 'pyproject.toml').write_text('[tool.poetry]\nname = "test"\n') + (tmp_path / 'poetry.lock').write_text(lock_content) + + doc = Document( + str(tmp_path / 'pyproject.toml'), + '[tool.poetry]\nname = "test"\n', + absolute_path=str(tmp_path / 'pyproject.toml'), + ) + result = restore_poetry.try_restore_dependencies(doc) + + assert result is not None + assert POETRY_LOCK_FILE_NAME in result.path + assert result.content == lock_content + + def test_get_lock_file_name(self, restore_poetry: RestorePoetryDependencies) -> None: + assert restore_poetry.get_lock_file_name() == POETRY_LOCK_FILE_NAME + + def test_get_commands_returns_poetry_lock(self, restore_poetry: RestorePoetryDependencies) -> None: + commands = restore_poetry.get_commands('/path/to/pyproject.toml') + assert commands == [['poetry', 'lock']] diff --git a/tests/cli/files_collector/test_commit_range_documents.py b/tests/cli/files_collector/test_commit_range_documents.py new file mode 100644 index 00000000..501c1811 --- /dev/null +++ b/tests/cli/files_collector/test_commit_range_documents.py @@ -0,0 +1,1166 @@ +import os +import tempfile +from collections.abc import Generator +from contextlib import contextmanager +from io import StringIO +from unittest.mock import Mock, patch + +import pytest +from git import Repo + +from cycode.cli import consts +from cycode.cli.files_collector.commit_range_documents import ( + _get_default_branches_for_merge_base, + calculate_pre_push_commit_range, + calculate_pre_receive_commit_range, + collect_commit_range_diff_documents, + get_diff_file_path, + get_safe_head_reference_for_diff, + parse_commit_range, + parse_pre_push_input, + parse_pre_receive_input, +) +from cycode.cli.utils.path_utils import get_path_by_os + +DUMMY_SHA_0 = '0' * 40 +DUMMY_SHA_1 = '1' * 40 +DUMMY_SHA_2 = '2' * 40 +DUMMY_SHA_A = 'a' * 40 +DUMMY_SHA_B = 'b' * 40 +DUMMY_SHA_C = 'c' * 40 + + +@contextmanager +def git_repository(path: str) -> Generator[Repo, None, None]: + """Context manager for Git repositories that ensures proper cleanup on Windows.""" + # Ensure the initialized repository uses 'main' as the default branch + repo = Repo.init(path, b='main') + try: + yield repo + finally: + # Properly close the repository to release file handles + repo.close() + + +@contextmanager +def temporary_git_repository() -> Generator[tuple[str, Repo], None, None]: + """Combined context manager for temporary directory with Git repository.""" + with tempfile.TemporaryDirectory() as temp_dir, git_repository(temp_dir) as repo: + yield temp_dir, repo + + +class TestGetSafeHeadReferenceForDiff: + """Test the safe HEAD reference functionality for git diff operations.""" + + def test_returns_head_when_repository_has_commits(self) -> None: + """Test that HEAD is returned when the repository has existing commits.""" + with temporary_git_repository() as (temp_dir, repo): + test_file = os.path.join(temp_dir, 'test.py') + with open(test_file, 'w') as f: + f.write("print('test')") + + repo.index.add(['test.py']) + repo.index.commit('Initial commit') + + result = get_safe_head_reference_for_diff(repo) + assert result == consts.GIT_HEAD_COMMIT_REV + + def test_returns_empty_tree_hash_when_repository_has_no_commits(self) -> None: + """Test that an empty tree hash is returned when the repository has no commits.""" + with temporary_git_repository() as (temp_dir, repo): + result = get_safe_head_reference_for_diff(repo) + expected_empty_tree_hash = consts.GIT_EMPTY_TREE_OBJECT + assert result == expected_empty_tree_hash + + +class TestIndexDiffWithSafeHeadReference: + """Test that index.diff works correctly with the safe head reference.""" + + def test_index_diff_works_on_bare_repository(self) -> None: + """Test that index.diff works on repositories with no commits.""" + with temporary_git_repository() as (temp_dir, repo): + test_file = os.path.join(temp_dir, 'staged_file.py') + with open(test_file, 'w') as f: + f.write("print('staged content')") + + repo.index.add(['staged_file.py']) + + head_ref = get_safe_head_reference_for_diff(repo) + diff_index = repo.index.diff(head_ref, create_patch=True, R=True) + + assert len(diff_index) == 1 + diff = diff_index[0] + assert diff.b_path == 'staged_file.py' + + def test_index_diff_works_on_repository_with_commits(self) -> None: + """Test that index.diff continues to work on repositories with existing commits.""" + with temporary_git_repository() as (temp_dir, repo): + initial_file = os.path.join(temp_dir, 'initial.py') + with open(initial_file, 'w') as f: + f.write("print('initial')") + + repo.index.add(['initial.py']) + repo.index.commit('Initial commit') + + new_file = os.path.join(temp_dir, 'new_file.py') + with open(new_file, 'w') as f: + f.write("print('new file')") + + with open(initial_file, 'w') as f: + f.write("print('modified initial')") + + repo.index.add(['new_file.py', 'initial.py']) + + head_ref = get_safe_head_reference_for_diff(repo) + diff_index = repo.index.diff(head_ref, create_patch=True, R=True) + + assert len(diff_index) == 2 + file_paths = {diff.b_path or diff.a_path for diff in diff_index} + assert 'new_file.py' in file_paths + assert 'initial.py' in file_paths + assert head_ref == consts.GIT_HEAD_COMMIT_REV + + def test_sequential_operations_on_same_repository(self) -> None: + """Test behavior when transitioning from bare to committed repository.""" + with temporary_git_repository() as (temp_dir, repo): + test_file = os.path.join(temp_dir, 'test.py') + with open(test_file, 'w') as f: + f.write("print('test')") + + repo.index.add(['test.py']) + + head_ref_before = get_safe_head_reference_for_diff(repo) + diff_before = repo.index.diff(head_ref_before, create_patch=True, R=True) + + expected_empty_tree = consts.GIT_EMPTY_TREE_OBJECT + assert head_ref_before == expected_empty_tree + assert len(diff_before) == 1 + + repo.index.commit('First commit') + + new_file = os.path.join(temp_dir, 'new.py') + with open(new_file, 'w') as f: + f.write("print('new')") + + repo.index.add(['new.py']) + + head_ref_after = get_safe_head_reference_for_diff(repo) + diff_after = repo.index.diff(head_ref_after, create_patch=True, R=True) + + assert head_ref_after == consts.GIT_HEAD_COMMIT_REV + assert len(diff_after) == 1 + assert diff_after[0].b_path == 'new.py' + + +def test_git_mv_pre_commit_scan() -> None: + with temporary_git_repository() as (temp_dir, repo): + newfile_path = os.path.join(temp_dir, 'NEWFILE.txt') + with open(newfile_path, 'w') as f: + f.write('test content') + + repo.index.add(['NEWFILE.txt']) + repo.index.commit('init') + + # Rename file but don't commit (this is the pre-commit scenario) + renamed_path = os.path.join(temp_dir, 'RENAMED.txt') + os.rename(newfile_path, renamed_path) + repo.index.remove(['NEWFILE.txt']) + repo.index.add(['RENAMED.txt']) + + head_ref = get_safe_head_reference_for_diff(repo) + diff_index = repo.index.diff(head_ref, create_patch=True, R=True) + + for diff in diff_index: + file_path = get_path_by_os(get_diff_file_path(diff, repo=repo)) + assert file_path == renamed_path + + +class TestGetDiffFilePath: + """Test the get_diff_file_path function with various diff scenarios.""" + + def test_diff_with_b_blob_and_working_tree(self) -> None: + """Test that blob.abspath is returned when b_blob is available and repo has a working tree.""" + with temporary_git_repository() as (temp_dir, repo): + test_file = os.path.join(temp_dir, 'test_file.py') + with open(test_file, 'w') as f: + f.write("print('original content')") + + repo.index.add(['test_file.py']) + repo.index.commit('Initial commit') + + with open(test_file, 'w') as f: + f.write("print('modified content')") + + repo.index.add(['test_file.py']) + + # Get diff of staged changes + head_ref = get_safe_head_reference_for_diff(repo) + diff_index = repo.index.diff(head_ref) + + assert len(diff_index) == 1 + + result = get_diff_file_path(diff_index[0], repo=repo) + + assert result == test_file + assert os.path.isabs(result) + + def test_diff_with_a_blob_only_and_working_tree(self) -> None: + """Test that a_blob.abspath is used when b_blob is None but a_blob exists.""" + with temporary_git_repository() as (temp_dir, repo): + test_file = os.path.join(temp_dir, 'to_delete.py') + with open(test_file, 'w') as f: + f.write("print('will be deleted')") + + repo.index.add(['to_delete.py']) + repo.index.commit('Initial commit') + + os.remove(test_file) + repo.index.remove(['to_delete.py']) + + # Get diff of staged changes + head_ref = get_safe_head_reference_for_diff(repo) + diff_index = repo.index.diff(head_ref) + + assert len(diff_index) == 1 + + result = get_diff_file_path(diff_index[0], repo=repo) + + assert result == test_file + assert os.path.isabs(result) + + def test_diff_with_b_path_fallback(self) -> None: + """Test that b_path is used with working_tree_dir when blob is not available.""" + with temporary_git_repository() as (temp_dir, repo): + test_file = os.path.join(temp_dir, 'new_file.py') + with open(test_file, 'w') as f: + f.write("print('new file')") + + repo.index.add(['new_file.py']) + + # for new files, there's no a_blob + head_ref = get_safe_head_reference_for_diff(repo) + diff_index = repo.index.diff(head_ref) + diff = diff_index[0] + + assert len(diff_index) == 1 + + result = get_diff_file_path(diff, repo=repo) + assert result == test_file + assert os.path.isabs(result) + + result = get_diff_file_path(diff, relative=True, repo=repo) + assert test_file.endswith(result) + assert not os.path.isabs(result) + + def test_diff_with_a_path_fallback(self) -> None: + """Test that a_path is used when b_path is None.""" + with temporary_git_repository() as (temp_dir, repo): + test_file = os.path.join(temp_dir, 'deleted_file.py') + with open(test_file, 'w') as f: + f.write("print('will be deleted')") + + repo.index.add(['deleted_file.py']) + repo.index.commit('Initial commit') + + # for deleted files, b_path might be None, so a_path should be used + os.remove(test_file) + repo.index.remove(['deleted_file.py']) + + head_ref = get_safe_head_reference_for_diff(repo) + diff_index = repo.index.diff(head_ref) + + assert len(diff_index) == 1 + diff = diff_index[0] + + result = get_diff_file_path(diff, repo=repo) + assert result == test_file + assert os.path.isabs(result) + + result = get_diff_file_path(diff, relative=True, repo=repo) + assert test_file.endswith(result) + assert not os.path.isabs(result) + + def test_diff_without_repo(self) -> None: + """Test behavior when repo is None.""" + with temporary_git_repository() as (temp_dir, repo): + test_file = os.path.join(temp_dir, 'test.py') + with open(test_file, 'w') as f: + f.write("print('test')") + + repo.index.add(['test.py']) + head_ref = get_safe_head_reference_for_diff(repo) + diff_index = repo.index.diff(head_ref) + + assert len(diff_index) == 1 + diff = diff_index[0] + + result = get_diff_file_path(diff, repo=None) + + expected_path = diff.b_path or diff.a_path + assert result == expected_path + assert not os.path.isabs(result) + + def test_diff_with_bare_repository(self) -> None: + """Test behavior when the repository has no working tree directory.""" + with tempfile.TemporaryDirectory() as temp_dir: + bare_repo = Repo.init(temp_dir, bare=True) + + try: + # Create a regular repo to push to the bare repo + with tempfile.TemporaryDirectory() as work_dir: + work_repo = Repo.init(work_dir, b='main') + try: + test_file = os.path.join(work_dir, 'test.py') + with open(test_file, 'w') as f: + f.write("print('test')") + + work_repo.index.add(['test.py']) + work_repo.index.commit('Initial commit') + + work_repo.create_remote('origin', temp_dir) + work_repo.remotes.origin.push('main:main') + + with open(test_file, 'w') as f: + f.write("print('modified')") + work_repo.index.add(['test.py']) + + # Get diff + diff_index = work_repo.index.diff('HEAD') + assert len(diff_index) == 1 + diff = diff_index[0] + + # Test with bare repo (no working_tree_dir) + result = get_diff_file_path(diff, repo=bare_repo) + + # Should return a relative path since bare repo has no working tree + expected_path = diff.b_path or diff.a_path + assert result == expected_path + assert not os.path.isabs(result) + finally: + work_repo.close() + finally: + bare_repo.close() + + def test_diff_with_no_paths(self) -> None: + """Test behavior when the diff has neither a_path nor b_path.""" + with temporary_git_repository() as (temp_dir, repo): + + class MockDiff: + def __init__(self) -> None: + self.a_path = None + self.b_path = None + self.a_blob = None + self.b_blob = None + + result = get_diff_file_path(MockDiff(), repo=repo) + assert result is None + + +class TestParsePrePushInput: + """Test the parse_pre_push_input function with various pre-push hook input scenarios.""" + + def test_parse_single_push_input(self) -> None: + """Test parsing a single branch push input.""" + pre_push_input = 'refs/heads/main 1234567890abcdef refs/heads/main 0987654321fedcba' + + with patch('sys.stdin', StringIO(pre_push_input)): + result = parse_pre_push_input() + assert result == 'refs/heads/main 1234567890abcdef refs/heads/main 0987654321fedcba' + + def test_parse_multiple_push_input_returns_first_line(self) -> None: + """Test parsing multiple branch push input returns only the first line.""" + pre_push_input = """refs/heads/main 1234567890abcdef refs/heads/main 0987654321fedcba +refs/heads/feature 1111111111111111 refs/heads/feature 2222222222222222""" + + with patch('sys.stdin', StringIO(pre_push_input)): + result = parse_pre_push_input() + assert result == 'refs/heads/main 1234567890abcdef refs/heads/main 0987654321fedcba' + + def test_parse_new_branch_push_input(self) -> None: + """Test parsing input for pushing a new branch (remote object name is all zeros).""" + pre_push_input = f'refs/heads/feature 1234567890abcdef refs/heads/feature {consts.EMPTY_COMMIT_SHA}' + + with patch('sys.stdin', StringIO(pre_push_input)): + result = parse_pre_push_input() + assert result == pre_push_input + + def test_parse_branch_deletion_input(self) -> None: + """Test parsing input for deleting a branch (local object name is all zeros).""" + pre_push_input = f'refs/heads/feature {consts.EMPTY_COMMIT_SHA} refs/heads/feature 1234567890abcdef' + + with patch('sys.stdin', StringIO(pre_push_input)): + result = parse_pre_push_input() + assert result == pre_push_input + + def test_parse_empty_input_raises_error(self) -> None: + """Test that empty input raises ValueError.""" + with patch('sys.stdin', StringIO('')), pytest.raises(ValueError, match='Pre push input was not found'): + parse_pre_push_input() + + def test_parse_whitespace_only_input_raises_error(self) -> None: + """Test that whitespace-only input raises ValueError.""" + with patch('sys.stdin', StringIO(' \n\t ')), pytest.raises(ValueError, match='Pre push input was not found'): + parse_pre_push_input() + + +class TestGetDefaultBranchesForMergeBase: + """Test the _get_default_branches_for_merge_base function with various scenarios.""" + + def test_environment_variable_override(self) -> None: + """Test that the environment variable takes precedence.""" + with ( + temporary_git_repository() as (temp_dir, repo), + patch.dict(os.environ, {consts.CYCODE_DEFAULT_BRANCH_ENV_VAR_NAME: 'custom-main'}), + ): + branches = _get_default_branches_for_merge_base(repo) + assert branches[0] == 'custom-main' + assert 'origin/main' in branches # Fallbacks should still be included + + def test_git_symbolic_ref_success(self) -> None: + """Test getting default branch via git symbolic-ref.""" + with temporary_git_repository() as (temp_dir, repo): + # Create a mock repo with a git interface that returns origin/main + mock_repo = Mock() + mock_repo.git.symbolic_ref.return_value = 'refs/remotes/origin/main' + + branches = _get_default_branches_for_merge_base(mock_repo) + assert 'origin/main' in branches + assert 'main' in branches + + def test_git_symbolic_ref_with_master(self) -> None: + """Test getting default branch via git symbolic-ref when it's master.""" + with temporary_git_repository() as (temp_dir, repo): + # Create a mock repo with a git interface that returns origin/master + mock_repo = Mock() + mock_repo.git.symbolic_ref.return_value = 'refs/remotes/origin/master' + + branches = _get_default_branches_for_merge_base(mock_repo) + assert 'origin/master' in branches + assert 'master' in branches + + def test_git_remote_show_fallback(self) -> None: + """Test fallback to git remote show when symbolic-ref fails.""" + with temporary_git_repository() as (temp_dir, repo): + # Create a mock repo where symbolic-ref fails but the remote show succeeds + mock_repo = Mock() + mock_repo.git.symbolic_ref.side_effect = Exception('symbolic-ref failed') + remote_output = """* remote origin + Fetch URL: https://github.com/user/repo.git + Push URL: https://github.com/user/repo.git + HEAD branch: develop + Remote branches: + develop tracked + main tracked""" + mock_repo.git.remote.return_value = remote_output + + branches = _get_default_branches_for_merge_base(mock_repo) + assert 'origin/develop' in branches + assert 'develop' in branches + + def test_both_git_methods_fail_fallback_to_hardcoded(self) -> None: + """Test fallback to hardcoded branches when both Git methods fail.""" + with temporary_git_repository() as (temp_dir, repo): + # Create a mock repo where both Git methods fail + mock_repo = Mock() + mock_repo.git.symbolic_ref.side_effect = Exception('symbolic-ref failed') + mock_repo.git.remote.side_effect = Exception('remote show failed') + + branches = _get_default_branches_for_merge_base(mock_repo) + # Should contain fallback branches + assert 'origin/main' in branches + assert 'origin/master' in branches + assert 'main' in branches + assert 'master' in branches + + def test_no_duplicates_in_branch_list(self) -> None: + """Test that duplicate branches are not added to the list.""" + with temporary_git_repository() as (temp_dir, repo): + # Create a mock repo that returns main (which is also in fallback list) + mock_repo = Mock() + mock_repo.git.symbolic_ref.return_value = 'refs/remotes/origin/main' + + branches = _get_default_branches_for_merge_base(mock_repo) + # Count occurrences of origin/main - should be exactly 1 + assert branches.count('origin/main') == 1 + assert branches.count('main') == 1 + + def test_env_var_plus_git_detection(self) -> None: + """Test combination of environment variable and git detection.""" + with temporary_git_repository() as (temp_dir, repo): + mock_repo = Mock() + mock_repo.git.symbolic_ref.return_value = 'refs/remotes/origin/develop' + + with patch.dict(os.environ, {consts.CYCODE_DEFAULT_BRANCH_ENV_VAR_NAME: 'origin/custom'}): + branches = _get_default_branches_for_merge_base(mock_repo) + # Env var should be first + assert branches[0] == 'origin/custom' + # Git detected branches should also be present + assert 'origin/develop' in branches + assert 'develop' in branches + + def test_malformed_symbolic_ref_response(self) -> None: + """Test handling of malformed symbolic-ref response.""" + with temporary_git_repository() as (temp_dir, repo): + # Create a mock repo that returns a malformed response + mock_repo = Mock() + mock_repo.git.symbolic_ref.return_value = 'malformed-response' + + branches = _get_default_branches_for_merge_base(mock_repo) + # Should fall back to hardcoded branches + assert 'origin/main' in branches + assert 'origin/master' in branches + + +class TestCalculatePrePushCommitRange: + """Test the calculate_pre_push_commit_range function with various Git repository scenarios.""" + + def test_calculate_range_for_existing_branch_update(self) -> None: + """Test calculating commit range for updating an existing branch.""" + push_details = 'refs/heads/main 1234567890abcdef refs/heads/main 0987654321fedcba' + + result = calculate_pre_push_commit_range(push_details) + assert result == '0987654321fedcba..1234567890abcdef' + + def test_calculate_range_for_branch_deletion_returns_none(self) -> None: + """Test that branch deletion returns None (no scanning needed).""" + push_details = f'refs/heads/feature {consts.EMPTY_COMMIT_SHA} refs/heads/feature 1234567890abcdef' + + result = calculate_pre_push_commit_range(push_details) + assert result is None + + def test_calculate_range_for_new_branch_with_merge_base(self) -> None: + """Test calculating commit range for a new branch when merge base is found.""" + with temporary_git_repository() as (temp_dir, repo): + # Create initial commit on main + test_file = os.path.join(temp_dir, 'main.py') + with open(test_file, 'w') as f: + f.write("print('main')") + + repo.index.add(['main.py']) + main_commit = repo.index.commit('Initial commit on main') + + # Create and switch to a feature branch + feature_branch = repo.create_head('feature') + feature_branch.checkout() + + # Add commits to a feature branch + feature_file = os.path.join(temp_dir, 'feature.py') + with open(feature_file, 'w') as f: + f.write("print('feature')") + + repo.index.add(['feature.py']) + feature_commit = repo.index.commit('Add feature') + + # Switch back to the default branch to simulate pushing the feature branch + repo.heads.main.checkout() + + # Test new branch push + push_details = f'refs/heads/feature {feature_commit.hexsha} refs/heads/feature {consts.EMPTY_COMMIT_SHA}' + + with patch('os.getcwd', return_value=temp_dir): + result = calculate_pre_push_commit_range(push_details) + assert result == f'{main_commit.hexsha}..{feature_commit.hexsha}' + + def test_calculate_range_for_new_branch_no_merge_base_fallback_to_all(self) -> None: + """Test that when no merge base is found, it falls back to scanning all commits.""" + with temporary_git_repository() as (temp_dir, repo): + # Create a single commit + test_file = os.path.join(temp_dir, 'test.py') + with open(test_file, 'w') as f: + f.write("print('test')") + + repo.index.add(['test.py']) + commit = repo.index.commit('Initial commit') + + # Test a new branch push with no default branch available + push_details = f'refs/heads/orphan {commit.hexsha} refs/heads/orphan {consts.EMPTY_COMMIT_SHA}' + + # Create a mock repo with a git interface that always raises exceptions for merge_base + mock_repo = Mock() + mock_git = Mock() + mock_git.merge_base.side_effect = Exception('No merge base found') + mock_repo.git = mock_git + + with ( + patch('os.getcwd', return_value=temp_dir), + patch('cycode.cli.files_collector.commit_range_documents.git_proxy.get_repo', return_value=mock_repo), + ): + result = calculate_pre_push_commit_range(push_details) + # Should fallback to --all when no merge base is found + assert result == '--all' + + def test_calculate_range_with_origin_main_as_merge_base(self) -> None: + """Test calculating commit range using origin/main as merge base.""" + with temporary_git_repository() as (temp_dir, repo): + # Create the main branch with commits + main_file = os.path.join(temp_dir, 'main.py') + with open(main_file, 'w') as f: + f.write("print('main')") + + repo.index.add(['main.py']) + main_commit = repo.index.commit('Main commit') + + # Create origin/main reference (simulating a remote) + repo.create_head('origin/main', main_commit) + + # Create feature branch from main + feature_branch = repo.create_head('feature', main_commit) + feature_branch.checkout() + + # Add feature commits + feature_file = os.path.join(temp_dir, 'feature.py') + with open(feature_file, 'w') as f: + f.write("print('feature')") + + repo.index.add(['feature.py']) + feature_commit = repo.index.commit('Feature commit') + + # Test new branch push + push_details = f'refs/heads/feature {feature_commit.hexsha} refs/heads/feature {consts.EMPTY_COMMIT_SHA}' + + with patch('os.getcwd', return_value=temp_dir): + result = calculate_pre_push_commit_range(push_details) + assert result == f'{main_commit.hexsha}..{feature_commit.hexsha}' + + def test_calculate_range_with_environment_variable_override(self) -> None: + """Test that environment variable override works for commit range calculation.""" + with temporary_git_repository() as (temp_dir, repo): + # Create custom default branch + custom_file = os.path.join(temp_dir, 'custom.py') + with open(custom_file, 'w') as f: + f.write("print('custom')") + + repo.index.add(['custom.py']) + custom_commit = repo.index.commit('Custom branch commit') + + # Create a custom branch + repo.create_head('custom-main', custom_commit) + + # Create a feature branch from custom + feature_branch = repo.create_head('feature', custom_commit) + feature_branch.checkout() + + # Add feature commits + feature_file = os.path.join(temp_dir, 'feature.py') + with open(feature_file, 'w') as f: + f.write("print('feature')") + + repo.index.add(['feature.py']) + feature_commit = repo.index.commit('Feature commit') + + # Test new branch push with custom default branch + push_details = f'refs/heads/feature {feature_commit.hexsha} refs/heads/feature {consts.EMPTY_COMMIT_SHA}' + + with ( + patch('os.getcwd', return_value=temp_dir), + patch.dict(os.environ, {consts.CYCODE_DEFAULT_BRANCH_ENV_VAR_NAME: 'custom-main'}), + ): + result = calculate_pre_push_commit_range(push_details) + assert result == f'{custom_commit.hexsha}..{feature_commit.hexsha}' + + def test_calculate_range_with_git_symbolic_ref_detection(self) -> None: + """Test commit range calculation with Git symbolic-ref detection.""" + with temporary_git_repository() as (temp_dir, repo): + # Create develop branch and commits + develop_file = os.path.join(temp_dir, 'develop.py') + with open(develop_file, 'w') as f: + f.write("print('develop')") + + repo.index.add(['develop.py']) + develop_commit = repo.index.commit('Develop commit') + + # Create origin/develop reference + repo.create_head('origin/develop', develop_commit) + repo.create_head('develop', develop_commit) + + # Create a feature branch + feature_branch = repo.create_head('feature', develop_commit) + feature_branch.checkout() + + # Add feature commits + feature_file = os.path.join(temp_dir, 'feature.py') + with open(feature_file, 'w') as f: + f.write("print('feature')") + + repo.index.add(['feature.py']) + feature_commit = repo.index.commit('Feature commit') + + # Test a new branch push with mocked default branch detection + push_details = f'refs/heads/feature {feature_commit.hexsha} refs/heads/feature {consts.EMPTY_COMMIT_SHA}' + + # # Mock the default branch detection to return origin/develop first + with ( + patch('os.getcwd', return_value=temp_dir), + patch( + 'cycode.cli.files_collector.commit_range_documents._get_default_branches_for_merge_base' + ) as mock_get_branches, + ): + mock_get_branches.return_value = [ + 'origin/develop', + 'develop', + 'origin/main', + 'main', + 'origin/master', + 'master', + ] + with patch('cycode.cli.files_collector.commit_range_documents.git_proxy.get_repo', return_value=repo): + result = calculate_pre_push_commit_range(push_details) + assert result == f'{develop_commit.hexsha}..{feature_commit.hexsha}' + + def test_calculate_range_with_origin_master_as_merge_base(self) -> None: + """Test calculating commit range using origin/master as a merge base.""" + with temporary_git_repository() as (temp_dir, repo): + # Create a main branch with commits + master_file = os.path.join(temp_dir, 'master.py') + with open(master_file, 'w') as f: + f.write("print('master')") + + repo.index.add(['master.py']) + master_commit = repo.index.commit('Master commit') + + # Create origin/master (master branch already exists by default) + repo.create_head('origin/master', master_commit) + + # Create a feature branch + feature_branch = repo.create_head('feature', master_commit) + feature_branch.checkout() + + # Add feature commits + feature_file = os.path.join(temp_dir, 'feature.py') + with open(feature_file, 'w') as f: + f.write("print('feature')") + + repo.index.add(['feature.py']) + feature_commit = repo.index.commit('Feature commit') + + # Test new branch push + push_details = f'refs/heads/feature {feature_commit.hexsha} refs/heads/feature {consts.EMPTY_COMMIT_SHA}' + + with patch('os.getcwd', return_value=temp_dir): + result = calculate_pre_push_commit_range(push_details) + assert result == f'{master_commit.hexsha}..{feature_commit.hexsha}' + + def test_calculate_range_exception_handling_fallback_to_all(self) -> None: + """Test that exceptions during Git repository access fall back to --all.""" + push_details = f'refs/heads/feature 1234567890abcdef refs/heads/feature {consts.EMPTY_COMMIT_SHA}' + + # Mock git_proxy.get_repo to raise an exception and capture the exception handling + with patch('cycode.cli.files_collector.commit_range_documents.git_proxy.get_repo') as mock_get_repo: + mock_get_repo.side_effect = Exception('Test exception') + result = calculate_pre_push_commit_range(push_details) + assert result == '--all' + + def test_calculate_range_parsing_push_details(self) -> None: + """Test that push details are correctly parsed into components.""" + # Test with standard format + push_details = 'refs/heads/feature abc123def456 refs/heads/feature 789xyz456abc' + + result = calculate_pre_push_commit_range(push_details) + assert result == '789xyz456abc..abc123def456' + + def test_calculate_range_with_tags(self) -> None: + """Test calculating commit range when pushing tags.""" + push_details = f'refs/tags/v1.0.0 1234567890abcdef refs/tags/v1.0.0 {consts.EMPTY_COMMIT_SHA}' + + with temporary_git_repository() as (temp_dir, repo): + # Create a commit + test_file = os.path.join(temp_dir, 'test.py') + with open(test_file, 'w') as f: + f.write("print('test')") + + repo.index.add(['test.py']) + commit = repo.index.commit('Test commit') + + # Create tag + repo.create_tag('v1.0.0', commit) + + with patch('os.getcwd', return_value=temp_dir): + result = calculate_pre_push_commit_range(push_details) + # For new tags, should try to find a merge base or fall back to --all + assert result in [f'{commit.hexsha}..{commit.hexsha}', '--all'] + + +class TestPrePushHookIntegration: + """Integration tests for pre-push hook functionality.""" + + def test_simulate_pre_push_hook_input_format(self) -> None: + """Test that our parsing handles the actual format Git sends to pre-push hooks.""" + # Simulate the exact format Git sends to pre-push hooks + test_cases = [ + # Standard branch push + 'refs/heads/main 67890abcdef12345 refs/heads/main 12345abcdef67890', + # New branch push + f'refs/heads/feature 67890abcdef12345 refs/heads/feature {consts.EMPTY_COMMIT_SHA}', + # Branch deletion + f'refs/heads/old-feature {consts.EMPTY_COMMIT_SHA} refs/heads/old-feature 12345abcdef67890', + # Tag push + f'refs/tags/v1.0.0 67890abcdef12345 refs/tags/v1.0.0 {consts.EMPTY_COMMIT_SHA}', + ] + + for push_input in test_cases: + with patch('sys.stdin', StringIO(push_input)): + parsed = parse_pre_push_input() + assert parsed == push_input + + # Test that we can calculate the commit range for each case + commit_range = calculate_pre_push_commit_range(parsed) + + if consts.EMPTY_COMMIT_SHA in push_input: + if push_input.startswith('refs/heads/') and push_input.split()[1] == consts.EMPTY_COMMIT_SHA: + # Branch deletion - should return None + assert commit_range is None + else: + # New branch/tag - should return a range or --all + assert commit_range is not None + else: + # Regular update - should return proper range + parts = push_input.split() + expected_range = f'{parts[3]}..{parts[1]}' + assert commit_range == expected_range + + +class TestParseCommitRange: + """Tests to validate unified parse_commit_range behavior matches git semantics.""" + + def _make_linear_history(self, repo: Repo, base_dir: str) -> tuple[str, str, str]: + """Create three linear commits A -> B -> C and return their SHAs.""" + a_file = os.path.join(base_dir, 'a.txt') + with open(a_file, 'w') as f: + f.write('A') + repo.index.add(['a.txt']) + a = repo.index.commit('A') + + with open(a_file, 'a') as f: + f.write('B') + repo.index.add(['a.txt']) + b = repo.index.commit('B') + + with open(a_file, 'a') as f: + f.write('C') + repo.index.add(['a.txt']) + c = repo.index.commit('C') + + return a.hexsha, b.hexsha, c.hexsha + + def test_two_dot_linear_history(self) -> None: + """For 'A..C', expect (A,C) in linear history.""" + with temporary_git_repository() as (temp_dir, repo): + a, b, c = self._make_linear_history(repo, temp_dir) + + parsed_from, parsed_to, separator = parse_commit_range(f'{a}..{c}', temp_dir) + assert (parsed_from, parsed_to, separator) == (a, c, '..') + + def test_three_dot_linear_history(self) -> None: + """For 'A...C' in linear history, expect (A,C).""" + with temporary_git_repository() as (temp_dir, repo): + a, b, c = self._make_linear_history(repo, temp_dir) + + parsed_from, parsed_to, separator = parse_commit_range(f'{a}...{c}', temp_dir) + assert (parsed_from, parsed_to, separator) == (a, c, '...') + + def test_open_right_linear_history(self) -> None: + """For 'A..', expect (A,HEAD=C).""" + with temporary_git_repository() as (temp_dir, repo): + a, b, c = self._make_linear_history(repo, temp_dir) + + parsed_from, parsed_to, separator = parse_commit_range(f'{a}..', temp_dir) + assert (parsed_from, parsed_to, separator) == (a, c, '..') + + def test_open_left_linear_history(self) -> None: + """For '..C' where HEAD==C, expect (HEAD=C,C).""" + with temporary_git_repository() as (temp_dir, repo): + a, b, c = self._make_linear_history(repo, temp_dir) + + parsed_from, parsed_to, separator = parse_commit_range(f'..{c}', temp_dir) + assert (parsed_from, parsed_to, separator) == (c, c, '..') + + def test_single_commit_spec(self) -> None: + """For 'A', expect (A,HEAD=C).""" + with temporary_git_repository() as (temp_dir, repo): + a, b, c = self._make_linear_history(repo, temp_dir) + + parsed_from, parsed_to, separator = parse_commit_range(a, temp_dir) + assert (parsed_from, parsed_to, separator) == (a, c, '..') + + def test_parse_all_for_empty_remote_scenario(self) -> None: + """Test that '--all' is parsed correctly for empty remote repository. + This repository has one commit locally. + """ + with temporary_git_repository() as (temp_dir, repo): + # Create a local commit (simulating first commit to empty remote) + test_file = os.path.join(temp_dir, 'test.py') + with open(test_file, 'w') as f: + f.write("print('test')") + + repo.index.add(['test.py']) + commit = repo.index.commit('Initial commit') + + # Test that '--all' (returned by calculate_pre_push_commit_range for empty remote) + # can be parsed to a valid commit range + parsed_from, parsed_to, separator = parse_commit_range('--all', temp_dir) + + # Should return first commit to HEAD (which is the only commit in this case) + assert parsed_from == commit.hexsha + assert parsed_to == commit.hexsha + assert separator == '..' + + def test_parse_all_for_empty_remote_scenario_with_two_commits(self) -> None: + """Test that '--all' is parsed correctly for empty remote repository. + This repository has two commits locally. + """ + with temporary_git_repository() as (temp_dir, repo): + # Create first commit + test_file = os.path.join(temp_dir, 'test.py') + with open(test_file, 'w') as f: + f.write("print('test')") + + repo.index.add(['test.py']) + commit1 = repo.index.commit('First commit') + + # Create second commit + test_file2 = os.path.join(temp_dir, 'test2.py') + with open(test_file2, 'w') as f: + f.write("print('test2')") + + repo.index.add(['test2.py']) + commit2 = repo.index.commit('Second commit') + + # Test that '--all' returns first commit to HEAD (second commit) + parsed_from, parsed_to, separator = parse_commit_range('--all', temp_dir) + + # Should return first commit to HEAD (second commit) + assert parsed_from == commit1.hexsha # First commit + assert parsed_to == commit2.hexsha # HEAD (second commit) + assert separator == '..' + + def test_parse_all_with_empty_repository_returns_none(self) -> None: + """Test that '--all' returns None when repository has no commits.""" + with temporary_git_repository() as (temp_dir, repo): + # Empty repository with no commits + parsed_from, parsed_to, separator = parse_commit_range('--all', temp_dir) + # Should return None, None, None when HEAD doesn't exist + assert parsed_from is None + assert parsed_to is None + assert separator is None + + +class TestParsePreReceiveInput: + """Test the parse_pre_receive_input function with various pre-receive hook input scenarios.""" + + def test_parse_single_update_input(self) -> None: + """Test parsing a single branch update input.""" + pre_receive_input = f'{DUMMY_SHA_1} {DUMMY_SHA_2} refs/heads/main' + + with patch('sys.stdin', StringIO(pre_receive_input)): + result = parse_pre_receive_input() + assert result == pre_receive_input + + def test_parse_multiple_update_input_returns_first_line(self) -> None: + """Test parsing multiple branch updates returns only the first line.""" + pre_receive_input = f"""{DUMMY_SHA_0} {DUMMY_SHA_A} refs/heads/main +{DUMMY_SHA_B} {DUMMY_SHA_C} refs/heads/feature""" + + with patch('sys.stdin', StringIO(pre_receive_input)): + result = parse_pre_receive_input() + assert result == f'{DUMMY_SHA_0} {DUMMY_SHA_A} refs/heads/main' + + def test_parse_empty_input_raises_error(self) -> None: + """Test that empty input raises ValueError.""" + match = 'Pre receive input was not found' + with patch('sys.stdin', StringIO('')), pytest.raises(ValueError, match=match): + parse_pre_receive_input() + + +class TestCalculatePreReceiveCommitRange: + """Test the calculate_pre_receive_commit_range function with representative scenarios.""" + + def test_branch_deletion_returns_none(self) -> None: + """When end commit is all zeros (deletion), no scan is needed.""" + update_details = f'{DUMMY_SHA_A} {consts.EMPTY_COMMIT_SHA} refs/heads/feature' + assert calculate_pre_receive_commit_range(os.getcwd(), update_details) is None + + def test_no_new_commits_returns_none(self) -> None: + """When there are no commits not in remote, return None.""" + with tempfile.TemporaryDirectory() as server_dir: + server_repo = Repo.init(server_dir, bare=True) + try: + with tempfile.TemporaryDirectory() as work_dir: + work_repo = Repo.init(work_dir, b='main') + try: + # Create a single commit and push it to the server as main (end commit is already on a ref) + test_file = os.path.join(work_dir, 'file.txt') + with open(test_file, 'w') as f: + f.write('base') + work_repo.index.add(['file.txt']) + end_commit = work_repo.index.commit('initial') + + work_repo.create_remote('origin', server_dir) + work_repo.remotes.origin.push('main:main') + + update_details = f'{DUMMY_SHA_A} {end_commit.hexsha} refs/heads/main' + assert calculate_pre_receive_commit_range(server_dir, update_details) is None + finally: + work_repo.close() + finally: + server_repo.close() + + def test_returns_triple_dot_range_from_oldest_unupdated(self) -> None: + """Returns '~1...' when there are new commits to scan.""" + with tempfile.TemporaryDirectory() as server_dir: + server_repo = Repo.init(server_dir, bare=True) + try: + with tempfile.TemporaryDirectory() as work_dir: + work_repo = Repo.init(work_dir, b='main') + try: + # Create commit A and push it to server as main (server has A on a ref) + a_path = os.path.join(work_dir, 'a.txt') + with open(a_path, 'w') as f: + f.write('A') + work_repo.index.add(['a.txt']) + work_repo.index.commit('A') + + work_repo.create_remote('origin', server_dir) + work_repo.remotes.origin.push('main:main') + + # Create commits B and C locally (not yet on server ref) + b_path = os.path.join(work_dir, 'b.txt') + with open(b_path, 'w') as f: + f.write('B') + work_repo.index.add(['b.txt']) + b_commit = work_repo.index.commit('B') + + c_path = os.path.join(work_dir, 'c.txt') + with open(c_path, 'w') as f: + f.write('C') + work_repo.index.add(['c.txt']) + end_commit = work_repo.index.commit('C') + + # Push the objects to a temporary ref and then delete that ref on server, + # so the objects exist but are not reachable from any ref. + work_repo.remotes.origin.push(f'{end_commit.hexsha}:refs/tmp/hold') + Repo(server_dir).git.update_ref('-d', 'refs/tmp/hold') + + update_details = f'{DUMMY_SHA_A} {end_commit.hexsha} refs/heads/main' + result = calculate_pre_receive_commit_range(server_dir, update_details) + assert result == f'{b_commit.hexsha}~1...{end_commit.hexsha}' + finally: + work_repo.close() + finally: + server_repo.close() + + def test_initial_oldest_commit_without_parent_returns_single_commit_range(self) -> None: + """If oldest commit has no parent, avoid '~1' and scan from end commit only.""" + with tempfile.TemporaryDirectory() as server_dir: + server_repo = Repo.init(server_dir, bare=True) + try: + with tempfile.TemporaryDirectory() as work_dir: + work_repo = Repo.init(work_dir, b='main') + try: + # Create a single root commit locally + p = os.path.join(work_dir, 'root.txt') + with open(p, 'w') as f: + f.write('root') + work_repo.index.add(['root.txt']) + end_commit = work_repo.index.commit('root') + + work_repo.create_remote('origin', server_dir) + # Push objects to a temporary ref and delete it so server has objects but no refs + work_repo.remotes.origin.push(f'{end_commit.hexsha}:refs/tmp/hold') + Repo(server_dir).git.update_ref('-d', 'refs/tmp/hold') + + update_details = f'{DUMMY_SHA_A} {end_commit.hexsha} refs/heads/main' + result = calculate_pre_receive_commit_range(server_dir, update_details) + assert result == end_commit.hexsha + finally: + work_repo.close() + finally: + server_repo.close() + + def test_initial_oldest_commit_without_parent_with_two_commits_returns_single_commit_range(self) -> None: + """If there are two new commits and the oldest has no parent, avoid '~1' and scan from end commit only.""" + with tempfile.TemporaryDirectory() as server_dir: + server_repo = Repo.init(server_dir, bare=True) + try: + with tempfile.TemporaryDirectory() as work_dir: + work_repo = Repo.init(work_dir, b='main') + try: + # Create two commits locally: oldest has no parent, second on top + a_path = os.path.join(work_dir, 'a.txt') + with open(a_path, 'w') as f: + f.write('A') + work_repo.index.add(['a.txt']) + work_repo.index.commit('A') + + d_path = os.path.join(work_dir, 'd.txt') + with open(d_path, 'w') as f: + f.write('D') + work_repo.index.add(['d.txt']) + end_commit = work_repo.index.commit('D') + + work_repo.create_remote('origin', server_dir) + # Push objects to a temporary ref and delete it so server has objects but no refs + work_repo.remotes.origin.push(f'{end_commit.hexsha}:refs/tmp/hold') + Repo(server_dir).git.update_ref('-d', 'refs/tmp/hold') + + update_details = f'{consts.EMPTY_COMMIT_SHA} {end_commit.hexsha} refs/heads/main' + result = calculate_pre_receive_commit_range(server_dir, update_details) + assert result == end_commit.hexsha + finally: + work_repo.close() + finally: + server_repo.close() + + +class TestCollectCommitRangeDiffDocuments: + """Test the collect_commit_range_diff_documents function with various commit range formats.""" + + def test_collect_with_various_commit_range_formats(self) -> None: + """Test that different commit range formats are normalized and work correctly.""" + with temporary_git_repository() as (temp_dir, repo): + # Create three commits + a_file = os.path.join(temp_dir, 'a.txt') + with open(a_file, 'w') as f: + f.write('A') + repo.index.add(['a.txt']) + a_commit = repo.index.commit('A') + + with open(a_file, 'a') as f: + f.write('B') + repo.index.add(['a.txt']) + b_commit = repo.index.commit('B') + + with open(a_file, 'a') as f: + f.write('C') + repo.index.add(['a.txt']) + c_commit = repo.index.commit('C') + + # Create mock context + mock_ctx = Mock() + mock_progress_bar = Mock() + mock_progress_bar.set_section_length = Mock() + mock_progress_bar.update = Mock() + mock_ctx.obj = {'progress_bar': mock_progress_bar} + + # Test two-dot range - should collect documents from commits B and C (2 commits, 2 documents) + commit_range = f'{a_commit.hexsha}..{c_commit.hexsha}' + documents = collect_commit_range_diff_documents(mock_ctx, temp_dir, commit_range) + assert len(documents) == 2, f'Expected 2 documents from range A..C, got {len(documents)}' + commit_ids_in_documents = {doc.unique_id for doc in documents if doc.unique_id} + assert b_commit.hexsha in commit_ids_in_documents + assert c_commit.hexsha in commit_ids_in_documents + + # Test three-dot range - should collect documents from commits B and C (2 commits, 2 documents) + commit_range = f'{a_commit.hexsha}...{c_commit.hexsha}' + documents = collect_commit_range_diff_documents(mock_ctx, temp_dir, commit_range) + assert len(documents) == 2, f'Expected 2 documents from range A...C, got {len(documents)}' + + # Test parent notation with three-dot - should collect document from commit C (1 commit, 1 document) + commit_range = f'{c_commit.hexsha}~1...{c_commit.hexsha}' + documents = collect_commit_range_diff_documents(mock_ctx, temp_dir, commit_range) + assert len(documents) == 1, f'Expected 1 document from range C~1...C, got {len(documents)}' + assert documents[0].unique_id == c_commit.hexsha + + # Test single commit spec - should be interpreted as A..HEAD (commits B and C, 2 documents) + commit_range = a_commit.hexsha + documents = collect_commit_range_diff_documents(mock_ctx, temp_dir, commit_range) + assert len(documents) == 2, f'Expected 2 documents from single commit A, got {len(documents)}' diff --git a/tests/cli/files_collector/test_documents_walk_ignore.py b/tests/cli/files_collector/test_documents_walk_ignore.py new file mode 100644 index 00000000..b92cb96e --- /dev/null +++ b/tests/cli/files_collector/test_documents_walk_ignore.py @@ -0,0 +1,430 @@ +import os +from os.path import normpath +from typing import TYPE_CHECKING + +from cycode.cli.files_collector.documents_walk_ignore import ( + _build_allowed_paths_set, + _create_ignore_filter_manager, + _filter_documents_by_allowed_paths, + _get_cycodeignore_path, + _get_document_check_path, + filter_documents_with_cycodeignore, +) +from cycode.cli.models import Document + +if TYPE_CHECKING: + from pyfakefs.fake_filesystem import FakeFilesystem + + +# we are using normpath() in every test to provide multi-platform support + + +def _create_mocked_file_structure(fs: 'FakeFilesystem') -> None: + """Create a mock file structure for testing.""" + fs.create_dir('/home/user/project') + fs.create_dir('/home/user/.git') + + fs.create_dir('/home/user/project/.cycode') + fs.create_file('/home/user/project/.cycode/config.yaml') + fs.create_dir('/home/user/project/.git') + fs.create_file('/home/user/project/.git/HEAD') + + # Create .cycodeignore with patterns + fs.create_file('/home/user/project/.cycodeignore', contents='*.pyc\n*.log\nbuild/\n# comment line\n\n') + + # Create test files that should be filtered + fs.create_file('/home/user/project/ignored.pyc') + fs.create_file('/home/user/project/ignored.log') + fs.create_file('/home/user/project/presented.txt') + fs.create_file('/home/user/project/presented.py') + + # Create build directory with files (should be ignored) + fs.create_dir('/home/user/project/build') + fs.create_file('/home/user/project/build/output.js') + fs.create_file('/home/user/project/build/bundle.css') + + # Create subdirectory + fs.create_dir('/home/user/project/src') + fs.create_file('/home/user/project/src/main.py') + fs.create_file('/home/user/project/src/debug.log') # should be ignored + fs.create_file('/home/user/project/src/temp.pyc') # should be ignored + + +def _create_test_documents(repo_path: str) -> list[Document]: + """Create test Document objects for the mocked file structure.""" + documents = [] + + # Files in root + documents.append( + Document( + path='ignored.pyc', + content='# compiled python', + absolute_path=normpath(os.path.join(repo_path, 'ignored.pyc')), + ) + ) + documents.append( + Document( + path='ignored.log', content='log content', absolute_path=normpath(os.path.join(repo_path, 'ignored.log')) + ) + ) + documents.append( + Document( + path='presented.txt', + content='text content', + absolute_path=normpath(os.path.join(repo_path, 'presented.txt')), + ) + ) + documents.append( + Document( + path='presented.py', + content='print("hello")', + absolute_path=normpath(os.path.join(repo_path, 'presented.py')), + ) + ) + + # Files in build directory (should be ignored) + documents.append( + Document( + path='build/output.js', + content='console.log("build");', + absolute_path=normpath(os.path.join(repo_path, 'build/output.js')), + ) + ) + documents.append( + Document( + path='build/bundle.css', + content='body { color: red; }', + absolute_path=normpath(os.path.join(repo_path, 'build/bundle.css')), + ) + ) + + # Files in src directory + documents.append( + Document( + path='src/main.py', + content='def main(): pass', + absolute_path=normpath(os.path.join(repo_path, 'src/main.py')), + ) + ) + documents.append( + Document( + path='src/debug.log', content='debug info', absolute_path=normpath(os.path.join(repo_path, 'src/debug.log')) + ) + ) + documents.append( + Document( + path='src/temp.pyc', content='compiled', absolute_path=normpath(os.path.join(repo_path, 'src/temp.pyc')) + ) + ) + + return documents + + +def test_get_cycodeignore_path() -> None: + """Test _get_cycodeignore_path helper function.""" + repo_path = normpath('/home/user/project') + expected = normpath('/home/user/project/.cycodeignore') + result = _get_cycodeignore_path(repo_path) + assert result == expected + + +def test_create_ignore_filter_manager(fs: 'FakeFilesystem') -> None: + """Test _create_ignore_filter_manager helper function.""" + _create_mocked_file_structure(fs) + + repo_path = normpath('/home/user/project') + cycodeignore_path = normpath('/home/user/project/.cycodeignore') + + manager = _create_ignore_filter_manager(repo_path, cycodeignore_path) + assert manager is not None + + # Test that it can walk the directory + walked_dirs = list(manager.walk_with_ignored()) + assert len(walked_dirs) > 0 + + +def test_get_document_check_path() -> None: + """Test _get_document_check_path helper function.""" + repo_path = normpath('/home/user/project') + + # Test document with absolute_path + doc_with_abs = Document( + path='src/main.py', content='code', absolute_path=normpath('/home/user/project/src/main.py') + ) + result = _get_document_check_path(doc_with_abs, repo_path) + assert result == normpath('/home/user/project/src/main.py') + + # Test document without absolute_path but with absolute path + doc_abs_path = Document(path=normpath('/home/user/project/src/main.py'), content='code') + result = _get_document_check_path(doc_abs_path, repo_path) + assert result == normpath('/home/user/project/src/main.py') + + # Test document with relative path + doc_rel_path = Document(path='src/main.py', content='code') + result = _get_document_check_path(doc_rel_path, repo_path) + assert result == normpath('/home/user/project/src/main.py') + + +def test_build_allowed_paths_set(fs: 'FakeFilesystem') -> None: + """Test _build_allowed_paths_set helper function.""" + _create_mocked_file_structure(fs) + + repo_path = normpath('/home/user/project') + cycodeignore_path = normpath('/home/user/project/.cycodeignore') + + manager = _create_ignore_filter_manager(repo_path, cycodeignore_path) + allowed_paths = _build_allowed_paths_set(manager, repo_path) + + # Check that allowed files are in the set + assert normpath('/home/user/project/presented.txt') in allowed_paths + assert normpath('/home/user/project/presented.py') in allowed_paths + assert normpath('/home/user/project/src/main.py') in allowed_paths + assert normpath('/home/user/project/.cycodeignore') in allowed_paths + + # Check that ignored files are NOT in the set + assert normpath('/home/user/project/ignored.pyc') not in allowed_paths + assert normpath('/home/user/project/ignored.log') not in allowed_paths + assert normpath('/home/user/project/src/debug.log') not in allowed_paths + assert normpath('/home/user/project/src/temp.pyc') not in allowed_paths + assert normpath('/home/user/project/build/output.js') not in allowed_paths + assert normpath('/home/user/project/build/bundle.css') not in allowed_paths + + +def test_filter_documents_by_allowed_paths() -> None: + """Test _filter_documents_by_allowed_paths helper function.""" + repo_path = normpath('/home/user/project') + + # Create test documents + documents = [ + Document(path='allowed.txt', content='content', absolute_path=normpath('/home/user/project/allowed.txt')), + Document(path='ignored.txt', content='content', absolute_path=normpath('/home/user/project/ignored.txt')), + ] + + # Create allowed paths set (only allow first document) + allowed_paths = {normpath('/home/user/project/allowed.txt')} + + result = _filter_documents_by_allowed_paths(documents, allowed_paths, repo_path) + + assert len(result) == 1 + assert result[0].path == 'allowed.txt' + + +def test_filter_documents_with_cycodeignore_no_ignore_file(fs: 'FakeFilesystem') -> None: + """Test filtering when no .cycodeignore file exists.""" + # Create structure without .cycodeignore + fs.create_dir('/home/user/project') + fs.create_file('/home/user/project/file1.py') + fs.create_file('/home/user/project/file2.log') + + repo_path = normpath('/home/user/project') + documents = [ + Document(path='file1.py', content='code'), + Document(path='file2.log', content='log'), + ] + + result = filter_documents_with_cycodeignore(documents, repo_path, is_cycodeignore_allowed=True) + + # Should return all documents since no .cycodeignore exists + assert len(result) == 2 + assert result == documents + + +def test_filter_documents_with_cycodeignore_basic_filtering(fs: 'FakeFilesystem') -> None: + """Test basic document filtering with .cycodeignore.""" + _create_mocked_file_structure(fs) + + repo_path = normpath('/home/user/project') + documents = _create_test_documents(repo_path) + + result = filter_documents_with_cycodeignore(documents, repo_path, is_cycodeignore_allowed=True) + + # Count expected results: should exclude *.pyc, *.log, and build/* files + expected_files = { + 'presented.txt', + 'presented.py', + 'src/main.py', + } + + result_files = {doc.path for doc in result} + assert result_files == expected_files + + # Verify specific exclusions + excluded_files = {doc.path for doc in documents if doc not in result} + assert 'ignored.pyc' in excluded_files + assert 'ignored.log' in excluded_files + assert 'src/debug.log' in excluded_files + assert 'src/temp.pyc' in excluded_files + assert 'build/output.js' in excluded_files + assert 'build/bundle.css' in excluded_files + + +def test_filter_documents_with_cycodeignore_relative_paths(fs: 'FakeFilesystem') -> None: + """Test filtering documents with relative paths (no absolute_path set).""" + _create_mocked_file_structure(fs) + + repo_path = normpath('/home/user/project') + + # Create documents without absolute_path + documents = [ + Document(path='presented.py', content='code'), + Document(path='ignored.pyc', content='compiled'), + Document(path='src/main.py', content='code'), + Document(path='src/debug.log', content='log'), + ] + + result = filter_documents_with_cycodeignore(documents, repo_path, is_cycodeignore_allowed=True) + + # Should filter out .pyc and .log files + expected_files = {'presented.py', 'src/main.py'} + result_files = {doc.path for doc in result} + assert result_files == expected_files + + +def test_filter_documents_with_cycodeignore_absolute_paths(fs: 'FakeFilesystem') -> None: + """Test filtering documents with absolute paths in path field.""" + _create_mocked_file_structure(fs) + + repo_path = normpath('/home/user/project') + + # Create documents with absolute paths in path field + documents = [ + Document(path=normpath('/home/user/project/presented.py'), content='code'), + Document(path=normpath('/home/user/project/ignored.pyc'), content='compiled'), + Document(path=normpath('/home/user/project/src/main.py'), content='code'), + Document(path=normpath('/home/user/project/src/debug.log'), content='log'), + ] + + result = filter_documents_with_cycodeignore(documents, repo_path, is_cycodeignore_allowed=True) + + # Should filter out .pyc and .log files + expected_files = {normpath('/home/user/project/presented.py'), normpath('/home/user/project/src/main.py')} + result_files = {doc.path for doc in result} + assert result_files == expected_files + + +def test_filter_documents_with_cycodeignore_empty_file(fs: 'FakeFilesystem') -> None: + """Test filtering with empty .cycodeignore file.""" + fs.create_dir('/home/user/project') + fs.create_file('/home/user/project/.cycodeignore', contents='') # empty file + fs.create_file('/home/user/project/file1.py') + fs.create_file('/home/user/project/file2.log') + + repo_path = normpath('/home/user/project') + documents = [ + Document(path='file1.py', content='code'), + Document(path='file2.log', content='log'), + ] + + result = filter_documents_with_cycodeignore(documents, repo_path, is_cycodeignore_allowed=True) + + # Should return all documents since .cycodeignore is empty + assert len(result) == 2 + + +def test_filter_documents_with_cycodeignore_comments_only(fs: 'FakeFilesystem') -> None: + """Test filtering with .cycodeignore file containing only comments and empty lines.""" + fs.create_dir('/home/user/project') + fs.create_file('/home/user/project/.cycodeignore', contents='# Just comments\n\n# More comments\n') + fs.create_file('/home/user/project/file1.py') + fs.create_file('/home/user/project/file2.log') + + repo_path = normpath('/home/user/project') + documents = [ + Document(path='file1.py', content='code'), + Document(path='file2.log', content='log'), + ] + + result = filter_documents_with_cycodeignore(documents, repo_path, is_cycodeignore_allowed=True) + + # Should return all documents since no real ignore patterns + assert len(result) == 2 + + +def test_filter_documents_with_cycodeignore_error_handling() -> None: + """Test error handling when document processing fails.""" + # Use non-existent repo path + repo_path = normpath('/non/existent/path') + + documents = [ + Document(path='file1.py', content='code'), + Document(path='file2.txt', content='content'), + ] + + # Should return all documents since .cycodeignore doesn't exist + result = filter_documents_with_cycodeignore(documents, repo_path, is_cycodeignore_allowed=True) + assert len(result) == 2 + + +def test_filter_documents_with_cycodeignore_complex_patterns(fs: 'FakeFilesystem') -> None: + """Test filtering with complex ignore patterns.""" + fs.create_dir('/home/user/project') + + # Create .cycodeignore with various pattern types + cycodeignore_content = """ +# Ignore specific files +config.json +secrets.key + +# Ignore file patterns +*.tmp +*.cache + +# Ignore directories +logs/ +temp/ + +# Ignore files in specific directories +tests/*.pyc +""" + fs.create_file('/home/user/project/.cycodeignore', contents=cycodeignore_content) + + # Create test files + test_files = [ + 'config.json', # ignored + 'secrets.key', # ignored + 'app.py', # allowed + 'file.tmp', # ignored + 'data.cache', # ignored + 'logs/app.log', # ignored (directory) + 'temp/file.txt', # ignored (directory) + 'tests/test.pyc', # ignored (pattern in directory) + 'tests/test.py', # allowed + 'src/main.py', # allowed + ] + + for file_path in test_files: + full_path = normpath(os.path.join('/home/user/project', file_path)) + os.makedirs(os.path.dirname(full_path), exist_ok=True) + fs.create_file(full_path) + + repo_path = normpath('/home/user/project') + documents = [Document(path=f, content='content') for f in test_files] + + result = filter_documents_with_cycodeignore(documents, repo_path, is_cycodeignore_allowed=True) + + # Should only allow: app.py, tests/test.py, src/main.py + expected_files = {'app.py', 'tests/test.py', 'src/main.py'} + result_files = {doc.path for doc in result} + assert result_files == expected_files + + +def test_filter_documents_with_cycodeignore_not_allowed(fs: 'FakeFilesystem') -> None: + """Test that filtering is skipped when is_cycodeignore_allowed is False.""" + _create_mocked_file_structure(fs) + + repo_path = normpath('/home/user/project') + documents = _create_test_documents(repo_path) + + # With filtering disabled, should return all documents + result = filter_documents_with_cycodeignore(documents, repo_path, is_cycodeignore_allowed=False) + + # Should return all documents without filtering + assert len(result) == len(documents) + assert {doc.path for doc in result} == {doc.path for doc in documents} + + # Verify that files that would normally be filtered are still present + result_files = {doc.path for doc in result} + assert 'ignored.pyc' in result_files + assert 'ignored.log' in result_files + assert 'build/output.js' in result_files + assert 'src/debug.log' in result_files diff --git a/tests/cli/files_collector/test_file_excluder.py b/tests/cli/files_collector/test_file_excluder.py new file mode 100644 index 00000000..31a52f55 --- /dev/null +++ b/tests/cli/files_collector/test_file_excluder.py @@ -0,0 +1,130 @@ +import pytest + +from cycode.cli import consts +from cycode.cli.files_collector.file_excluder import Excluder, _is_file_relevant_for_sca_scan + + +class TestIsFileRelevantForScaScan: + """Test the SCA path exclusion logic.""" + + def test_files_in_excluded_directories_should_be_excluded(self) -> None: + """Test that files inside excluded directories are properly excluded.""" + + # Test node_modules exclusion + assert _is_file_relevant_for_sca_scan('project/node_modules/package/index.js') is False + assert _is_file_relevant_for_sca_scan('/project/node_modules/package.json') is False + assert _is_file_relevant_for_sca_scan('deep/nested/node_modules/lib/file.txt') is False + + # Test .gradle exclusion + assert _is_file_relevant_for_sca_scan('project/.gradle/wrapper/gradle-wrapper.jar') is False + assert _is_file_relevant_for_sca_scan('/home/user/.gradle/caches/modules.xml') is False + + # Test venv exclusion + assert _is_file_relevant_for_sca_scan('project/venv/lib/python3.8/site-packages/module.py') is False + assert _is_file_relevant_for_sca_scan('/home/user/venv/bin/activate') is False + + # Test __pycache__ exclusion + assert _is_file_relevant_for_sca_scan('src/__pycache__/module.cpython-38.pyc') is False + assert _is_file_relevant_for_sca_scan('project/utils/__pycache__/helper.pyc') is False + + def test_files_with_excluded_names_in_filename_should_be_included(self) -> None: + """Test that files containing excluded directory names in their filename are NOT excluded.""" + + # These should be INCLUDED because the excluded terms are in the filename, not directory path + assert _is_file_relevant_for_sca_scan('project/build.gradle') is True + assert _is_file_relevant_for_sca_scan('project/gradlew') is True + assert _is_file_relevant_for_sca_scan('app/node_modules_backup.txt') is True + assert _is_file_relevant_for_sca_scan('src/venv_setup.py') is True + assert _is_file_relevant_for_sca_scan('utils/pycache_cleaner.py') is True + assert _is_file_relevant_for_sca_scan('config/gradle_config.xml') is True + + def test_files_with_excluded_extensions_in_should_be_included(self) -> None: + """Test that files containing excluded extensions are NOT excluded.""" + excluder = Excluder() + # These should be INCLUDED because the excluded terms are in the filename + assert excluder._is_relevant_file_to_scan_common('iac', 'project/cfg/Dockerfile') is True + assert excluder._is_relevant_file_to_scan_common('iac', 'project/cfg/build.tf') is True + assert excluder._is_relevant_file_to_scan_common('iac', 'project/cfg/build.tf.json') is True + assert excluder._is_relevant_file_to_scan_common('iac', 'project/cfg/config.json') is True + assert excluder._is_relevant_file_to_scan_common('iac', 'project/cfg/config.yaml') is True + assert excluder._is_relevant_file_to_scan_common('iac', 'project/cfg/config.yml') is True + # These should be EXCLUDED because the excluded terms are not in the filename + assert excluder._is_relevant_file_to_scan_common('iac', 'project/cfg/build') is False + assert excluder._is_relevant_file_to_scan_common('iac', 'project/cfg/build') is False + assert excluder._is_relevant_file_to_scan_common('iac', 'project/cfg/Dockerfile.txt') is False + assert excluder._is_relevant_file_to_scan_common('iac', 'project/cfg/config.ini') is False + + def test_files_in_regular_directories_should_be_included(self) -> None: + """Test that files in regular directories (not excluded) are included.""" + + assert _is_file_relevant_for_sca_scan('project/src/main.py') is True + assert _is_file_relevant_for_sca_scan('app/components/button.tsx') is True + assert _is_file_relevant_for_sca_scan('/home/user/project/package.json') is True + assert _is_file_relevant_for_sca_scan('build/dist/app.js') is True + assert _is_file_relevant_for_sca_scan('tests/unit/test_utils.py') is True + + def test_multiple_excluded_directories_in_path(self) -> None: + """Test paths that contain multiple excluded directories.""" + + # Should be excluded if ANY directory in the path is excluded + assert _is_file_relevant_for_sca_scan('project/venv/lib/node_modules/package.json') is False + assert _is_file_relevant_for_sca_scan('app/node_modules/dep/.gradle/build.xml') is False + assert _is_file_relevant_for_sca_scan('src/__pycache__/nested/venv/file.py') is False + + def test_absolute_vs_relative_paths(self) -> None: + """Test both absolute and relative path formats.""" + + # Relative paths + assert _is_file_relevant_for_sca_scan('node_modules/package.json') is False + assert _is_file_relevant_for_sca_scan('src/app.py') is True + + # Absolute paths + assert _is_file_relevant_for_sca_scan('/home/user/project/node_modules/lib.js') is False + assert _is_file_relevant_for_sca_scan('/home/user/project/src/main.py') is True + + def test_edge_cases(self) -> None: + """Test edge cases and boundary conditions.""" + + # Empty string should be considered relevant (no path to exclude) + assert _is_file_relevant_for_sca_scan('') is True + # Single filename without a directory + assert _is_file_relevant_for_sca_scan('package.json') is True + # Root-level excluded directory + assert _is_file_relevant_for_sca_scan('/node_modules/package.json') is False + # Excluded directory as part of the filename but in allowed directory + assert _is_file_relevant_for_sca_scan('src/my_node_modules_file.js') is True + + def test_case_sensitivity(self) -> None: + """Test that directory matching is case-sensitive.""" + + # Excluded directories are lowercase, so uppercase versions should be included + assert _is_file_relevant_for_sca_scan('project/NODE_MODULES/package.json') is True + assert _is_file_relevant_for_sca_scan('project/Node_Modules/lib.js') is True + assert _is_file_relevant_for_sca_scan('project/VENV/lib/module.py') is True + + # But exact case matches should be excluded + assert _is_file_relevant_for_sca_scan('project/node_modules/package.json') is False + assert _is_file_relevant_for_sca_scan('project/venv/lib/module.py') is False + + def test_nested_excluded_directories(self) -> None: + """Test deeply nested directory structures with excluded directories.""" + + # Deep nesting should still work + deep_path = 'a/b/c/d/e/f/g/node_modules/h/i/j/package.json' + assert _is_file_relevant_for_sca_scan(deep_path) is False + + # Multiple levels of excluded directories + multi_excluded = 'project/node_modules/package/venv/lib/__pycache__/module.pyc' + assert _is_file_relevant_for_sca_scan(multi_excluded) is False + + @pytest.mark.parametrize('excluded_dir', consts.SCA_EXCLUDED_FOLDER_IN_PATH) + def test_parametrized_excluded_directories(self, excluded_dir: str) -> None: + """Parametrized test to ensure all excluded directories work correctly.""" + + # File inside excluded directory should be excluded + excluded_path = f'project/{excluded_dir}/file.txt' + assert _is_file_relevant_for_sca_scan(excluded_path) is False + + # File with excluded directory name in filename should be included + included_path = f'project/src/{excluded_dir}_config.txt' + assert _is_file_relevant_for_sca_scan(included_path) is True diff --git a/tests/cli/files_collector/test_in_memory_zip.py b/tests/cli/files_collector/test_in_memory_zip.py new file mode 100644 index 00000000..d1790c7c --- /dev/null +++ b/tests/cli/files_collector/test_in_memory_zip.py @@ -0,0 +1,29 @@ +"""Tests for InMemoryZip class, specifically for handling surrogate characters and encoding issues.""" + +import zipfile +from io import BytesIO + +from cycode.cli.files_collector.models.in_memory_zip import InMemoryZip + + +def test_append_with_surrogate_characters() -> None: + """Test that surrogate characters are handled gracefully without raising encoding errors.""" + # Surrogate characters (U+D800 to U+DFFF) cannot be encoded to UTF-8 directly + zip_file = InMemoryZip() + content = 'Normal text \udc96 more text' + + # Should not raise UnicodeEncodeError + zip_file.append('test.txt', None, content) + zip_file.close() + + # Verify the ZIP was created successfully + zip_data = zip_file.read() + assert len(zip_data) > 0 + + # Verify we can read it back and the surrogate was replaced + with zipfile.ZipFile(BytesIO(zip_data), 'r') as zf: + extracted = zf.read('test.txt').decode('utf-8') + assert 'Normal text' in extracted + assert 'more text' in extracted + # The surrogate should have been replaced with the replacement character + assert '\udc96' not in extracted diff --git a/tests/cli/files_collector/test_walk_ignore.py b/tests/cli/files_collector/test_walk_ignore.py new file mode 100644 index 00000000..ffcf46cb --- /dev/null +++ b/tests/cli/files_collector/test_walk_ignore.py @@ -0,0 +1,173 @@ +import os +from os.path import normpath +from typing import TYPE_CHECKING + +from cycode.cli.files_collector.walk_ignore import ( + _collect_top_level_ignore_files, + _walk_to_top, + walk_ignore, +) + +if TYPE_CHECKING: + from pyfakefs.fake_filesystem import FakeFilesystem + + +# we are using normpath() in every test to provide multi-platform support + + +def test_walk_to_top() -> None: + path = normpath('/a/b/c/d/e/f/g') + result = list(_walk_to_top(path)) + assert result == [ + normpath('/a/b/c/d/e/f/g'), + normpath('/a/b/c/d/e/f'), + normpath('/a/b/c/d/e'), + normpath('/a/b/c/d'), + normpath('/a/b/c'), + normpath('/a/b'), + normpath('/a'), + normpath('/'), + ] + + path = normpath('/a') + result = list(_walk_to_top(path)) + assert result == [normpath('/a'), normpath('/')] + + path = normpath('/') + result = list(_walk_to_top(path)) + assert result == [normpath('/')] + + path = normpath('a') + result = list(_walk_to_top(path)) + assert result == [normpath('a')] + + +def _create_mocked_file_structure(fs: 'FakeFilesystem') -> None: + fs.create_dir('/home/user/project') + fs.create_dir('/home/user/.git') + + fs.create_dir('/home/user/project/.cycode') + fs.create_file('/home/user/project/.cycode/config.yaml') + fs.create_dir('/home/user/project/.git') + fs.create_file('/home/user/project/.git/HEAD') + + fs.create_file('/home/user/project/.gitignore', contents='*.pyc\n*.log') + fs.create_file('/home/user/project/.cycodeignore', contents='*.cs') + fs.create_file('/home/user/project/ignored.pyc') + fs.create_file('/home/user/project/ignored.cs') + fs.create_file('/home/user/project/presented.txt') + fs.create_file('/home/user/project/ignored2.log') + fs.create_file('/home/user/project/ignored2.pyc') + fs.create_file('/home/user/project/ignored2.cs') + fs.create_file('/home/user/project/presented2.txt') + + fs.create_dir('/home/user/project/subproject') + fs.create_file('/home/user/project/subproject/.gitignore', contents='*.txt') + fs.create_file('/home/user/project/subproject/.cycodeignore', contents='*.cs') + fs.create_file('/home/user/project/subproject/ignored.txt') + fs.create_file('/home/user/project/subproject/ignored.log') + fs.create_file('/home/user/project/subproject/ignored.pyc') + fs.create_file('/home/user/project/subproject/ignored.cs') + fs.create_file('/home/user/project/subproject/presented.py') + + +def test_collect_top_level_ignore_files(fs: 'FakeFilesystem') -> None: + _create_mocked_file_structure(fs) + + # Test with path inside the project + path = normpath('/home/user/project/subproject') + ignore_files = _collect_top_level_ignore_files(path) + assert len(ignore_files) == 4 + assert normpath('/home/user/project/.gitignore') in ignore_files + assert normpath('/home/user/project/subproject/.gitignore') in ignore_files + assert normpath('/home/user/project/.cycodeignore') in ignore_files + assert normpath('/home/user/project/subproject/.cycodeignore') in ignore_files + + # Test with path at the top level with no ignore files + path = normpath('/home/user/.git') + ignore_files = _collect_top_level_ignore_files(path) + assert len(ignore_files) == 0 + + # Test with path at the top level with ignore files + path = normpath('/home/user/project') + ignore_files = _collect_top_level_ignore_files(path) + assert len(ignore_files) == 2 + assert normpath('/home/user/project/.gitignore') in ignore_files + assert normpath('/home/user/project/.cycodeignore') in ignore_files + + # Test with a path that does not have any ignore files + fs.remove('/home/user/project/.gitignore') + fs.remove('/home/user/project/.cycodeignore') + path = normpath('/home/user') + ignore_files = _collect_top_level_ignore_files(path) + assert len(ignore_files) == 0 + fs.create_file('/home/user/project/.gitignore', contents='*.pyc\n*.log') + + +def _collect_walk_ignore_files(path: str) -> list[str]: + files = [] + for root, _, filenames in walk_ignore(path): + for filename in filenames: + files.append(os.path.join(root, filename)) + + return files + + +def test_walk_ignore(fs: 'FakeFilesystem') -> None: + _create_mocked_file_structure(fs) + + path = normpath('/home/user/project') + result = _collect_walk_ignore_files(path) + + assert len(result) == 7 + # ignored globally by default: + assert normpath('/home/user/project/.git/HEAD') not in result + assert normpath('/home/user/project/.cycode/config.yaml') not in result + # ignored by .gitignore in project directory: + assert normpath('/home/user/project/ignored.pyc') not in result + assert normpath('/home/user/project/subproject/ignored.pyc') not in result + # ignored by .cycodeignore in project directory: + assert normpath('/home/user/project/ignored.cs') not in result + assert normpath('/home/user/project/subproject/ignored.cs') not in result + # ignored by .gitignore in subproject directory: + assert normpath('/home/user/project/subproject/ignored.txt') not in result + # ignored by .cycodeignore in project directory: + assert normpath('/home/user/project/ignored2.log') not in result + assert normpath('/home/user/project/ignored2.pyc') not in result + assert normpath('/home/user/project/subproject/ignored.log') not in result + # ignored by .cycodeignore in subproject directory: + assert normpath('/home/user/project/ignored2.cs') not in result + # presented after both .gitignore and .cycodeignore: + assert normpath('/home/user/project/.gitignore') in result + assert normpath('/home/user/project/subproject/.gitignore') in result + assert normpath('/home/user/project/presented.txt') in result + assert normpath('/home/user/project/presented2.txt') in result + assert normpath('/home/user/project/subproject/presented.py') in result + + path = normpath('/home/user/project/subproject') + result = _collect_walk_ignore_files(path) + + assert len(result) == 3 + # ignored: + assert normpath('/home/user/project/subproject/ignored.txt') not in result + assert normpath('/home/user/project/subproject/ignored.log') not in result + assert normpath('/home/user/project/subproject/ignored.pyc') not in result + # presented: + assert normpath('/home/user/project/subproject/presented.py') in result + + +def test_walk_ignore_top_level_ignores_order(fs: 'FakeFilesystem') -> None: + fs.create_file('/home/user/.gitignore', contents='*.log') + fs.create_file('/home/user/project/.gitignore', contents='!*.log') # rollback *.log ignore for project + fs.create_dir('/home/user/project/subproject') + + fs.create_file('/home/user/ignored.log') + fs.create_file('/home/user/project/presented.log') + fs.create_file('/home/user/project/subproject/presented.log') + + path = normpath('/home/user/project') + result = _collect_walk_ignore_files(path) + assert len(result) == 3 + assert normpath('/home/user/ignored.log') not in result + assert normpath('/home/user/project/presented.log') in result + assert normpath('/home/user/project/subproject/presented.log') in result diff --git a/tests/cli/models/__init__.py b/tests/cli/models/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cli/models/test_severity.py b/tests/cli/models/test_severity.py new file mode 100644 index 00000000..a59d5751 --- /dev/null +++ b/tests/cli/models/test_severity.py @@ -0,0 +1,11 @@ +from cycode.cli.cli_types import SeverityOption + + +def test_get_member_weight() -> None: + assert SeverityOption.get_member_weight('INFO') == 0 + assert SeverityOption.get_member_weight('LOW') == 1 + assert SeverityOption.get_member_weight('MEDIUM') == 2 + assert SeverityOption.get_member_weight('HIGH') == 3 + assert SeverityOption.get_member_weight('CRITICAL') == 4 + + assert SeverityOption.get_member_weight('NON_EXISTENT') == -1 diff --git a/tests/cli/printers/__init__.py b/tests/cli/printers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cli/printers/utils/__init__.py b/tests/cli/printers/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cli/printers/utils/test_detection_data.py b/tests/cli/printers/utils/test_detection_data.py new file mode 100644 index 00000000..603c25db --- /dev/null +++ b/tests/cli/printers/utils/test_detection_data.py @@ -0,0 +1,41 @@ +from pathlib import Path +from unittest.mock import MagicMock + +from cycode.cli.consts import IAC_SCAN_TYPE, SAST_SCAN_TYPE, SCA_SCAN_TYPE, SECRET_SCAN_TYPE +from cycode.cli.printers.utils.detection_data import get_detection_file_path + + +def _make_detection(**details: str) -> MagicMock: + detection = MagicMock() + detection.detection_details = dict(details) + return detection + + +def test_get_detection_file_path_sca_uses_file_path() -> None: + detection = _make_detection(file_name='package.json', file_path='/repo/path/package.json') + result = get_detection_file_path(SCA_SCAN_TYPE, detection) + assert result == Path('/repo/path/package.json') + + +def test_get_detection_file_path_iac_uses_file_path() -> None: + detection = _make_detection(file_name='main.tf', file_path='/repo/infra/main.tf') + result = get_detection_file_path(IAC_SCAN_TYPE, detection) + assert result == Path('/repo/infra/main.tf') + + +def test_get_detection_file_path_sca_fallback_empty() -> None: + detection = _make_detection() + result = get_detection_file_path(SCA_SCAN_TYPE, detection) + assert result == Path('') + + +def test_get_detection_file_path_secret() -> None: + detection = _make_detection(file_path='/repo/src', file_name='.env') + result = get_detection_file_path(SECRET_SCAN_TYPE, detection) + assert result == Path('/repo/src/.env') + + +def test_get_detection_file_path_sast() -> None: + detection = _make_detection(file_path='repo/src/app.py') + result = get_detection_file_path(SAST_SCAN_TYPE, detection) + assert result == Path('/repo/src/app.py') diff --git a/tests/cli/printers/utils/test_rich_encoding_fix.py b/tests/cli/printers/utils/test_rich_encoding_fix.py new file mode 100644 index 00000000..721f1c6a --- /dev/null +++ b/tests/cli/printers/utils/test_rich_encoding_fix.py @@ -0,0 +1,86 @@ +"""Tests for Rich encoding fix to handle surrogate characters.""" + +from io import StringIO +from typing import Any +from unittest.mock import MagicMock + +from rich.console import Console + +from cycode.cli import consts +from cycode.cli.models import Document +from cycode.cli.printers.rich_printer import RichPrinter +from cycode.cyclient.models import Detection + + +def create_strict_encoding_console() -> tuple[Console, StringIO]: + """Create a Console that enforces strict UTF-8 encoding, simulating Windows console behavior. + + When Rich writes to the console, the file object needs to encode strings to bytes. + With errors='strict' (default for TextIOWrapper), this raises UnicodeEncodeError on surrogates. + This function simulates that behavior to test the encoding fix. + """ + buffer = StringIO() + + class StrictEncodingWrapper: + def __init__(self, file_obj: StringIO) -> None: + self._file = file_obj + + def write(self, text: str) -> int: + """Validate encoding before writing to simulate strict encoding behavior.""" + text.encode('utf-8') + return self._file.write(text) + + def flush(self) -> None: + self._file.flush() + + def isatty(self) -> bool: + return False + + def __getattr__(self, name: str) -> Any: + # Delegate all other attributes to the underlying file + return getattr(self._file, name) + + strict_file = StrictEncodingWrapper(buffer) + console = Console(file=strict_file, width=80, force_terminal=False) + return console, buffer + + +def test_rich_printer_handles_surrogate_characters_in_violation_card() -> None: + """Test that RichPrinter._print_violation_card() handles surrogate characters without errors. + + The error occurs in Rich's console._write_buffer() -> write() when console.print() is called. + On Windows with strict encoding, this raises UnicodeEncodeError on surrogates. + """ + surrogate_char = chr(0xDC96) + document_content = 'A' * 1236 + surrogate_char + 'B' * 100 + document = Document( + path='test.py', + content=document_content, + is_git_diff_format=False, + ) + + detection = Detection( + detection_type_id='test-id', + type='test-type', + message='Test message', + detection_details={ + 'description': 'Summary with ' + surrogate_char + ' surrogate character', + 'policy_display_name': 'Test Policy', + 'start_position': 1236, + 'length': 1, + 'line': 0, + }, + detection_rule_id='test-rule-id', + severity='Medium', + ) + + mock_ctx = MagicMock() + mock_ctx.obj = { + 'scan_type': consts.SAST_SCAN_TYPE, + 'show_secret': False, + } + mock_ctx.info_name = consts.SAST_SCAN_TYPE + + console, _ = create_strict_encoding_console() + printer = RichPrinter(mock_ctx, console, console) + printer._print_violation_card(document, detection, 1, 1) diff --git a/tests/cli/test_code_scanner.py b/tests/cli/test_code_scanner.py deleted file mode 100644 index ee4e19bb..00000000 --- a/tests/cli/test_code_scanner.py +++ /dev/null @@ -1,73 +0,0 @@ -import os - -import click -import pytest -from click import ClickException -from git import InvalidGitRepositoryError -from requests import Response - -from cycode.cli.code_scanner import _handle_exception, _is_file_relevant_for_sca_scan, exclude_irrelevant_files # noqa -from cycode.cli.exceptions import custom_exceptions - - -@pytest.fixture() -def ctx(): - return click.Context(click.Command('path'), obj={'verbose': False, 'output': 'text'}) - - -@pytest.mark.parametrize('exception, expected_soft_fail', [ - (custom_exceptions.NetworkError(400, 'msg', Response()), True), - (custom_exceptions.ScanAsyncError('msg'), True), - (custom_exceptions.HttpUnauthorizedError('msg', Response()), True), - (custom_exceptions.ZipTooLargeError(1000), True), - (InvalidGitRepositoryError(), None), -]) -def test_handle_exception_soft_fail( - ctx: click.Context, exception: custom_exceptions.CycodeError, expected_soft_fail: bool): - with ctx: - _handle_exception(ctx, exception) - - assert ctx.obj.get('did_fail') is True - assert ctx.obj.get('soft_fail') is expected_soft_fail - - -def test_handle_exception_unhandled_error(ctx: click.Context): - with ctx: - with pytest.raises(ClickException): - _handle_exception(ctx, ValueError('test')) - - assert ctx.obj.get('did_fail') is True - assert ctx.obj.get('soft_fail') is None - - -def test_handle_exception_click_error(ctx: click.Context): - with ctx: - with pytest.raises(ClickException): - _handle_exception(ctx, click.ClickException('test')) - - assert ctx.obj.get('did_fail') is True - assert ctx.obj.get('soft_fail') is None - - -def test_handle_exception_verbose(monkeypatch): - ctx = click.Context(click.Command('path'), obj={'verbose': True, 'output': 'text'}) - - def mock_secho(msg, *_, **__): - assert 'Error:' in msg - - monkeypatch.setattr(click, 'secho', mock_secho) - - with ctx: - with pytest.raises(ClickException): - _handle_exception(ctx, ValueError('test')) - - -def test_is_file_relevant_for_sca_scan(): - path = os.path.join('some_package', 'node_modules', 'package.json') - assert _is_file_relevant_for_sca_scan(path) is False - path = os.path.join('some_package', 'node_modules', 'package.lock') - assert _is_file_relevant_for_sca_scan(path) is False - path = os.path.join('some_package', 'package.json') - assert _is_file_relevant_for_sca_scan(path) is True - path = os.path.join('some_package', 'package.lock') - assert _is_file_relevant_for_sca_scan(path) is True diff --git a/tests/cli/test_main.py b/tests/cli/test_main.py deleted file mode 100644 index e018819f..00000000 --- a/tests/cli/test_main.py +++ /dev/null @@ -1,61 +0,0 @@ -import json - -import pytest -from typing import TYPE_CHECKING - -import responses -from click.testing import CliRunner - -from cycode.cli.main import main_cli -from tests.conftest import TEST_FILES_PATH, CLI_ENV_VARS -from tests.cyclient.test_scan_client import get_zipped_file_scan_response, get_zipped_file_scan_url - -_PATH_TO_SCAN = TEST_FILES_PATH.joinpath('zip_content').absolute() - -if TYPE_CHECKING: - from cycode.cyclient.scan_client import ScanClient - - -def _is_json(plain: str) -> bool: - try: - json.loads(plain) - return True - except (ValueError, TypeError): - return False - - -@responses.activate -@pytest.mark.parametrize('output', ['text', 'json']) -@pytest.mark.parametrize('option_space', ['scan', 'global']) -def test_passing_output_option( - output: str, option_space: str, scan_client: 'ScanClient', api_token_response: responses.Response -): - scan_type = 'secret' - - responses.add(get_zipped_file_scan_response(get_zipped_file_scan_url(scan_type, scan_client))) - responses.add(api_token_response) - # scan report is not mocked. This raise connection error on attempt to report scan. it doesn't perform real request - - args = ['scan', '--soft-fail', 'path', str(_PATH_TO_SCAN)] - - if option_space == 'global': - global_args = ['--output', output] - global_args.extend(args) - - args = global_args - elif option_space == 'scan': - # test backward compatability with old style command - args.insert(2, '--output') - args.insert(3, output) - - result = CliRunner().invoke(main_cli, args, env=CLI_ENV_VARS) - - except_json = output == 'json' - - assert _is_json(result.output) == except_json - - if except_json: - output = json.loads(result.output) - assert 'scan_id' in output - else: - assert 'Scan Results' in result.output diff --git a/tests/conftest.py b/tests/conftest.py index 3af4b303..f1df29dd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,33 +1,66 @@ from pathlib import Path +from typing import Optional import pytest import responses +from cycode.cli.user_settings.credentials_manager import CredentialsManager +from cycode.cyclient.client_creator import create_scan_client +from cycode.cyclient.cycode_oidc_based_client import CycodeOidcBasedClient from cycode.cyclient.cycode_token_based_client import CycodeTokenBasedClient from cycode.cyclient.scan_client import ScanClient -from cycode.cyclient.scan_config.scan_config_creator import create_scan_client -_EXPECTED_API_TOKEN = 'someJWT' +# not real JWT with userId and tenantId fields +_EXPECTED_API_TOKEN = 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyLCJ1c2VySWQiOiJibGFibGEiLCJ0ZW5hbnRJZCI6ImJsYWJsYSJ9.8RfoWBfciuj8nwc7UB8uOUJchVuaYpYlgf1G2QHiWTk' # noqa: E501 _CLIENT_ID = 'b1234568-0eaa-1234-beb8-6f0c12345678' _CLIENT_SECRET = 'a12345a-42b2-1234-3bdd-c0130123456' +_ID_TOKEN = 'eyJhbGciOiJSUzI1NiIsImtpZCI6IjEyMzQ1NiJ9.eyJzdWIiOiI4NzY1NDMyMSIsImF1ZCI6ImN5Y29kZSIsImV4cCI6MTUxNjIzOTAyMiwiaXNfb2lkYyI6MX0.Rrby2hPzsoMM3' # noqa: E501 -CLI_ENV_VARS = { - 'CYCODE_CLIENT_ID': _CLIENT_ID, - 'CYCODE_CLIENT_SECRET': _CLIENT_SECRET -} +CLI_ENV_VARS = {'CYCODE_CLIENT_ID': _CLIENT_ID, 'CYCODE_CLIENT_SECRET': _CLIENT_SECRET} TEST_FILES_PATH = Path(__file__).parent.joinpath('test_files').absolute() +MOCKED_RESPONSES_PATH = Path(__file__).parent.joinpath('cyclient/mocked_responses/data').absolute() +ZIP_CONTENT_PATH = TEST_FILES_PATH.joinpath('zip_content').absolute() + + +@pytest.fixture(scope='session') +def test_files_path() -> Path: + return TEST_FILES_PATH @pytest.fixture(scope='session') def scan_client() -> ScanClient: - return create_scan_client(_CLIENT_ID, _CLIENT_SECRET) + return create_scan_client(_CLIENT_ID, _CLIENT_SECRET, hide_response_log=False) + + +def create_token_based_client( + client_id: Optional[str] = None, client_secret: Optional[str] = None +) -> CycodeTokenBasedClient: + CredentialsManager.FILE_NAME = 'unit-tests-credentials.yaml' + + if client_id is None: + client_id = _CLIENT_ID + if client_secret is None: + client_secret = _CLIENT_SECRET + + return CycodeTokenBasedClient(client_id, client_secret) + + +def create_oidc_based_client(client_id: Optional[str] = None, id_token: Optional[str] = None) -> CycodeOidcBasedClient: + CredentialsManager.FILE_NAME = 'unit-tests-credentials.yaml' + + if client_id is None: + client_id = _CLIENT_ID + if id_token is None: + id_token = _ID_TOKEN + + return CycodeOidcBasedClient(client_id, id_token) @pytest.fixture(scope='session') def token_based_client() -> CycodeTokenBasedClient: - return CycodeTokenBasedClient(_CLIENT_ID, _CLIENT_SECRET) + return create_token_based_client() @pytest.fixture(scope='session') @@ -36,16 +69,16 @@ def api_token_url(token_based_client: CycodeTokenBasedClient) -> str: @pytest.fixture(scope='session') -def api_token_response(api_token_url) -> responses.Response: +def api_token_response(api_token_url: str) -> responses.Response: return responses.Response( method=responses.POST, url=api_token_url, json={ 'token': _EXPECTED_API_TOKEN, 'refresh_token': '12345678-0c68-1234-91ba-a13123456789', - 'expires_in': 86400 + 'expires_in': 86400, }, - status=200 + status=200, ) @@ -53,4 +86,35 @@ def api_token_response(api_token_url) -> responses.Response: @responses.activate def api_token(token_based_client: CycodeTokenBasedClient, api_token_response: responses.Response) -> str: responses.add(api_token_response) - return token_based_client.api_token + return token_based_client.get_access_token() + + +@pytest.fixture(scope='session') +def oidc_based_client() -> CycodeOidcBasedClient: + return create_oidc_based_client() + + +@pytest.fixture(scope='session') +def oidc_api_token_url(oidc_based_client: CycodeOidcBasedClient) -> str: + return f'{oidc_based_client.api_url}/api/v1/auth/oidc/api-token' + + +@pytest.fixture(scope='session') +@responses.activate +def oidc_api_token(oidc_based_client: CycodeOidcBasedClient, oidc_api_token_response: responses.Response) -> str: + responses.add(oidc_api_token_response) + return oidc_based_client.get_access_token() + + +@pytest.fixture(scope='session') +def oidc_api_token_response(oidc_api_token_url: str) -> responses.Response: + return responses.Response( + method=responses.POST, + url=oidc_api_token_url, + json={ + 'token': _EXPECTED_API_TOKEN, + 'refresh_token': '12345678-0c68-1234-91ba-a13123456789', + 'expires_in': 86400, + }, + status=200, + ) diff --git a/tests/cyclient/mocked_responses/__init__.py b/tests/cyclient/mocked_responses/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cyclient/mocked_responses/data/detection_rules.json b/tests/cyclient/mocked_responses/data/detection_rules.json new file mode 100644 index 00000000..b221951a --- /dev/null +++ b/tests/cyclient/mocked_responses/data/detection_rules.json @@ -0,0 +1,20 @@ +[ + { + "classification_data": [ + { + "severity": "High", + "classification_rule_id": "e4e826bd-5820-4cc9-ae5b-bbfceb500a21" + } + ], + "detection_rule_id": "26ab3395-2522-4061-a50a-c69c2d622ca1" + }, + { + "classification_data": [ + { + "severity": "High", + "classification_rule_id": "e4e826bd-5820-4cc9-ae5b-bbfceb500a22" + } + ], + "detection_rule_id": "12345678-aea1-4304-a6e9-012345678901" + } +] diff --git a/tests/cyclient/mocked_responses/data/detections.json b/tests/cyclient/mocked_responses/data/detections.json new file mode 100644 index 00000000..89196489 --- /dev/null +++ b/tests/cyclient/mocked_responses/data/detections.json @@ -0,0 +1,83 @@ +[ + { + "source_policy_name": "Secrets detection", + "source_policy_type": "SensitiveContent", + "source_entity_name": null, + "source_entity_id": null, + "detection_type_id": "7dff932a-418f-47ee-abb0-703e0f6592cd", + "root_id": null, + "status": "Open", + "status_updated_at": null, + "status_reason": null, + "status_change_message": null, + "source_entity_type": "Audit", + "detection_details": { + "organization_name": null, + "organization_id": "", + "sha512": "6e6c867188c04340d9ecfa1b7e56a356e605f2a70fbda865f11b4a57eb07e634", + "provider": "CycodeCli", + "concrete_provider": "CycodeCli", + "length": 55, + "start_position": 19, + "line": 0, + "commit_id": null, + "member_id": "", + "member_name": "", + "member_email": "", + "author_name": "", + "author_email": "", + "branch_name": "", + "committer_name": "", + "committed_at": "0001-01-01T00:00:00+00:00", + "file_path": "%FILEPATH%", + "file_name": "secrets.py", + "file_extension": ".py", + "url": null, + "should_resolve_upon_branch_deletion": false, + "position_in_line": 19, + "repository_name": null, + "repository_id": null, + "old_detection_id": "f35c42f99d3712d4593d5ee16a9ceb36ca9fb20b33e68edd0e00847e6a02a7b6" + }, + "severity": "Medium", + "remediable": false, + "correlation_message": "Secret of type 'Slack Token' was found in filename 'secrets.py' within '' repository", + "provider": "CycodeCli", + "scan_id": "%SCAN_ID%", + "assignee_id": null, + "type": "slack-token", + "is_hidden": false, + "tags": [], + "detection_rule_id": "26ab3395-2522-4061-a50a-c69c2d622ca1", + "classification": null, + "priority": 0, + "metadata": null, + "labels": [], + "detection_id": "f35c42f99d3712d4593d5ee16a9ceb36ca9fb20b33e68edd0e00847e6a02a7b6", + "internal_note": null, + "sdlc_stages": [ + "Code", + "Container Registry", + "Productivity Tools", + "Cloud", + "Build" + ], + "policy_labels": [], + "category": "SecretDetection", + "sub_category": "SensitiveContent", + "sub_category_v2": "Messaging Systems", + "policy_tags": [], + "remediations": [], + "instruction_details": { + "instruction_name_to_single_id_map": null, + "instruction_name_to_multiple_ids_map": null, + "instruction_tags": null + }, + "external_detection_references": [], + "project_ids": [], + "tenant_id": "123456-663f-4e27-9170-e559c2379292", + "id": "123456-895a-4830-b6c1-b948e99b71a4", + "created_date": "2023-10-25T11:07:14.7516793+00:00", + "updated_date": "2023-10-25T11:07:14.7616063+00:00" + } +] \ No newline at end of file diff --git a/tests/cyclient/mocked_responses/import_sbom_client.py b/tests/cyclient/mocked_responses/import_sbom_client.py new file mode 100644 index 00000000..04398ace --- /dev/null +++ b/tests/cyclient/mocked_responses/import_sbom_client.py @@ -0,0 +1,67 @@ +from typing import Optional + +import responses +from responses import matchers + +from cycode.cyclient.import_sbom_client import ImportSbomClient + + +def get_import_sbom_url(import_sbom_client: ImportSbomClient) -> str: + api_url = import_sbom_client.client.api_url + service_url = ImportSbomClient.IMPORT_SBOM_REQUEST_PATH + return f'{api_url}/{service_url}' + + +def get_import_sbom_response(url: str, status: int = 201) -> responses.Response: + json_response = {'message': 'SBOM imported successfully'} + return responses.Response(method=responses.POST, url=url, json=json_response, status=status) + + +def get_member_details_url(import_sbom_client: ImportSbomClient) -> str: + api_url = import_sbom_client.client.api_url + service_url = ImportSbomClient.GET_USER_ID_REQUEST_PATH + return f'{api_url}/{service_url}' + + +def get_member_details_response( + url: str, email: str, external_id: Optional[str] = None, status: int = 200 +) -> responses.Response: + items = [] + if external_id: + items = [{'external_id': external_id, 'email': email}] + + json_response = { + 'items': items, + 'page_size': 10, + 'next_page_token': None, + } + + return responses.Response( + method=responses.GET, + url=url, + json=json_response, + status=status, + match=[matchers.query_param_matcher({'email': email})], + ) + + +def mock_import_sbom_responses( + responses_module: responses, + import_sbom_client: ImportSbomClient, + status: int = 201, +) -> None: + """Mock the basic SBOM import endpoint""" + responses_module.add(get_import_sbom_response(get_import_sbom_url(import_sbom_client), status)) + + +def mock_member_details_response( + responses_module: responses, + import_sbom_client: ImportSbomClient, + email: str, + external_id: Optional[str] = None, + status: int = 200, +) -> None: + """Mock the member details lookup endpoint""" + responses_module.add( + get_member_details_response(get_member_details_url(import_sbom_client), email, external_id, status) + ) diff --git a/tests/cyclient/mocked_responses/scan_client.py b/tests/cyclient/mocked_responses/scan_client.py new file mode 100644 index 00000000..c37c1d8a --- /dev/null +++ b/tests/cyclient/mocked_responses/scan_client.py @@ -0,0 +1,145 @@ +import json +from pathlib import Path +from typing import Optional +from uuid import UUID, uuid4 + +import responses + +from cycode.cyclient.scan_client import ScanClient +from tests.conftest import MOCKED_RESPONSES_PATH + + +def get_zipped_file_scan_async_url(scan_type: str, scan_client: ScanClient) -> str: + api_url = scan_client.scan_cycode_client.api_url + service_url = scan_client.get_zipped_file_scan_async_url_path(scan_type) + return f'{api_url}/{service_url}' + + +def get_zipped_file_scan_async_response(url: str, scan_id: Optional[UUID] = None) -> responses.Response: + if not scan_id: + scan_id = uuid4() + + json_response = { + 'scan_id': str(scan_id), # not always as expected due to _get_scan_id and passing scan_id to cxt of CLI + } + + return responses.Response(method=responses.POST, url=url, json=json_response, status=200) + + +def get_scan_details_url(scan_type: str, scan_id: Optional[UUID], scan_client: ScanClient) -> str: + api_url = scan_client.scan_cycode_client.api_url + service_url = scan_client.get_scan_details_path(scan_type, str(scan_id)) + return f'{api_url}/{service_url}' + + +def get_scan_aggregation_report_url(aggregation_id: Optional[UUID], scan_client: ScanClient, scan_type: str) -> str: + api_url = scan_client.scan_cycode_client.api_url + service_url = scan_client.get_scan_aggregation_report_url_path(str(aggregation_id), scan_type) + return f'{api_url}/{service_url}' + + +def get_scan_aggregation_report_url_response(url: str, aggregation_id: Optional[UUID] = None) -> responses.Response: + if not aggregation_id: + aggregation_id = uuid4() + json_response = {'report_url': f'https://app.domain/cli-logs-aggregation/{aggregation_id}'} + + return responses.Response(method=responses.GET, url=url, json=json_response, status=200) + + +def get_scan_details_response(url: str, scan_id: Optional[UUID] = None) -> responses.Response: + if not scan_id: + scan_id = uuid4() + + json_response = { + 'id': str(scan_id), + 'scan_type': 'Secrets', + 'metadata': f'Path: {scan_id}, Folder: scans, Size: 465', + 'entity_type': 'ZippedFile', + 'entity_id': 'Repository', + 'parent_entity_id': 'Organization', + 'organization_id': 'Organization', + 'scm_provider': 'CycodeCli', + 'started_at': '2023-10-25T10:02:23.048282+00:00', + 'finished_at': '2023-10-25T10:02:26.867082+00:00', + 'scan_status': 'Completed', # mark as completed to avoid mocking repeated requests + 'message': None, + 'results_count': 1, + 'scan_update_at': '2023-10-25T10:02:26.867216+00:00', + 'duration': {'days': 0, 'hours': 0, 'minutes': 0, 'seconds': 3, 'milliseconds': 818}, + 'is_hidden': False, + 'is_initial_scan': False, + 'detection_messages': [], + } + + return responses.Response(method=responses.GET, url=url, json=json_response, status=200) + + +def get_scan_detections_url(scan_client: ScanClient) -> str: + api_url = scan_client.scan_cycode_client.api_url + path = scan_client.get_scan_detections_list_path() + return f'{api_url}/{path}' + + +def get_scan_detections_response(url: str, scan_id: UUID, zip_content_path: Path) -> responses.Response: + with open(MOCKED_RESPONSES_PATH.joinpath('detections.json'), encoding='UTF-8') as f: + content = f.read() + content = content.replace('%FILEPATH%', str(zip_content_path.absolute().as_posix())) + content = content.replace('%SCAN_ID%', str(scan_id)) + + json_response = json.loads(content) + + return responses.Response(method=responses.GET, url=url, json=json_response, status=200) + + +def get_report_scan_status_url(scan_type: str, scan_id: UUID, scan_client: ScanClient) -> str: + api_url = scan_client.scan_cycode_client.api_url + service_url = scan_client.get_report_scan_status_path(scan_type, str(scan_id)) + return f'{api_url}/{service_url}' + + +def get_report_scan_status_response(url: str) -> responses.Response: + return responses.Response(method=responses.POST, url=url, status=200) + + +def get_detection_rules_url(scan_client: ScanClient) -> str: + api_url = scan_client.scan_cycode_client.api_url + service_url = scan_client.get_detection_rules_path() + return f'{api_url}/{service_url}' + + +def get_detection_rules_response(url: str) -> responses.Response: + with open(MOCKED_RESPONSES_PATH.joinpath('detection_rules.json'), encoding='UTF-8') as f: + json_response = json.load(f) + + return responses.Response(method=responses.GET, url=url, json=json_response, status=200) + + +def get_scan_configuration_url(scan_type: str, scan_client: ScanClient) -> str: + api_url = scan_client.scan_cycode_client.api_url + service_url = scan_client.get_scan_configuration_path(scan_type) + return f'{api_url}/{service_url}' + + +def get_scan_configuration_response(url: str) -> responses.Response: + json_response = { + 'scannable_extensions': None, + } + + return responses.Response(method=responses.GET, url=url, json=json_response, status=200) + + +def mock_remote_config_responses(responses_module: responses, scan_type: str, scan_client: ScanClient) -> None: + responses_module.add(get_scan_configuration_response(get_scan_configuration_url(scan_type, scan_client))) + + +def mock_scan_async_responses( + responses_module: responses, scan_type: str, scan_client: ScanClient, scan_id: UUID, zip_content_path: Path +) -> None: + mock_remote_config_responses(responses_module, scan_type, scan_client) + responses_module.add( + get_zipped_file_scan_async_response(get_zipped_file_scan_async_url(scan_type, scan_client), scan_id) + ) + responses_module.add(get_scan_details_response(get_scan_details_url(scan_type, scan_id, scan_client), scan_id)) + responses_module.add(get_detection_rules_response(get_detection_rules_url(scan_client))) + responses_module.add(get_scan_detections_response(get_scan_detections_url(scan_client), scan_id, zip_content_path)) + responses_module.add(get_report_scan_status_response(get_report_scan_status_url(scan_type, scan_id, scan_client))) diff --git a/tests/cyclient/scan_config/test_default_scan_config.py b/tests/cyclient/scan_config/test_default_scan_config.py index 1043f505..987c6c78 100644 --- a/tests/cyclient/scan_config/test_default_scan_config.py +++ b/tests/cyclient/scan_config/test_default_scan_config.py @@ -1,22 +1,17 @@ -from cycode.cyclient.scan_config.scan_config_creator import DefaultScanConfig +from cycode.cli import consts +from cycode.cyclient.scan_config_base import DefaultScanConfig -def test_get_service_name(): +def test_get_service_name() -> None: default_scan_config = DefaultScanConfig() - assert default_scan_config.get_service_name('secret') == 'secret' - assert default_scan_config.get_service_name('iac') == 'iac' - assert default_scan_config.get_service_name('sca') == 'scans' - assert default_scan_config.get_service_name('sast') == 'scans' + assert default_scan_config.get_service_name(consts.SECRET_SCAN_TYPE) == 'scans' + assert default_scan_config.get_service_name(consts.IAC_SCAN_TYPE) == 'scans' + assert default_scan_config.get_service_name(consts.SCA_SCAN_TYPE) == 'scans' + assert default_scan_config.get_service_name(consts.SAST_SCAN_TYPE) == 'scans' -def test_get_scans_prefix(): - default_scan_config = DefaultScanConfig() - - assert default_scan_config.get_scans_prefix() == 'scans' - - -def test_get_detections_prefix(): +def test_get_detections_prefix() -> None: default_scan_config = DefaultScanConfig() assert default_scan_config.get_detections_prefix() == 'detections' diff --git a/tests/cyclient/scan_config/test_dev_scan_config.py b/tests/cyclient/scan_config/test_dev_scan_config.py index 28f42c63..f1cd484c 100644 --- a/tests/cyclient/scan_config/test_dev_scan_config.py +++ b/tests/cyclient/scan_config/test_dev_scan_config.py @@ -1,22 +1,17 @@ -from cycode.cyclient.scan_config.scan_config_creator import DevScanConfig +from cycode.cli import consts +from cycode.cyclient.scan_config_base import DevScanConfig -def test_get_service_name(): +def test_get_service_name() -> None: dev_scan_config = DevScanConfig() - assert dev_scan_config.get_service_name('secret') == '5025' - assert dev_scan_config.get_service_name('iac') == '5026' - assert dev_scan_config.get_service_name('sca') == '5004' - assert dev_scan_config.get_service_name('sast') == '5004' + assert dev_scan_config.get_service_name(consts.SECRET_SCAN_TYPE) == '5004' + assert dev_scan_config.get_service_name(consts.IAC_SCAN_TYPE) == '5004' + assert dev_scan_config.get_service_name(consts.SCA_SCAN_TYPE) == '5004' + assert dev_scan_config.get_service_name(consts.SAST_SCAN_TYPE) == '5004' -def test_get_scans_prefix(): - dev_scan_config = DevScanConfig() - - assert dev_scan_config.get_scans_prefix() == '5004' - - -def test_get_detections_prefix(): +def test_get_detections_prefix() -> None: dev_scan_config = DevScanConfig() assert dev_scan_config.get_detections_prefix() == '5016' diff --git a/tests/cyclient/test_auth_client.py b/tests/cyclient/test_auth_client.py index a74b2ec7..24d9b096 100644 --- a/tests/cyclient/test_auth_client.py +++ b/tests/cyclient/test_auth_client.py @@ -3,22 +3,24 @@ import responses from requests import Timeout +from cycode.cli.apps.auth.auth_manager import AuthManager +from cycode.cli.exceptions.custom_exceptions import CycodeError, RequestTimeoutError from cycode.cyclient.auth_client import AuthClient -from cycode.cyclient.models import AuthenticationSession, ApiTokenGenerationPollingResponse, \ - ApiTokenGenerationPollingResponseSchema -from cycode.cli.exceptions.custom_exceptions import CycodeError +from cycode.cyclient.models import ( + ApiTokenGenerationPollingResponse, + ApiTokenGenerationPollingResponseSchema, + AuthenticationSession, +) @pytest.fixture(scope='module') def code_challenge() -> str: - from cycode.cli.auth.auth_manager import AuthManager code_challenge, _ = AuthManager()._generate_pkce_code_pair() return code_challenge @pytest.fixture(scope='module') def code_verifier() -> str: - from cycode.cli.auth.auth_manager import AuthManager _, code_verifier = AuthManager()._generate_pkce_code_pair() return code_verifier @@ -31,25 +33,19 @@ def auth_client() -> AuthClient: @pytest.fixture(scope='module', name='start_url') def auth_start_url(client: AuthClient) -> str: # TODO(MarshalX): create database of constants of endpoints. remove hardcoded paths - return client.cycode_client.build_full_url( - client.cycode_client.api_url, - f'{client.AUTH_CONTROLLER_PATH}/start' - ) + return client.cycode_client.build_full_url(client.cycode_client.api_url, f'{client.AUTH_CONTROLLER_PATH}/start') @pytest.fixture(scope='module', name='token_url') def auth_token_url(client: AuthClient) -> str: - return client.cycode_client.build_full_url( - client.cycode_client.api_url, - f'{client.AUTH_CONTROLLER_PATH}/token' - ) + return client.cycode_client.build_full_url(client.cycode_client.api_url, f'{client.AUTH_CONTROLLER_PATH}/token') _SESSION_ID = '4cff1234-a209-47ed-ab2f-85676912345c' @responses.activate -def test_start_session_success(client: AuthClient, start_url: str, code_challenge: str): +def test_start_session_success(client: AuthClient, start_url: str, code_challenge: str) -> None: responses.add( responses.POST, start_url, @@ -63,7 +59,7 @@ def test_start_session_success(client: AuthClient, start_url: str, code_challeng @responses.activate -def test_start_session_timeout(client: AuthClient, start_url: str, code_challenge: str): +def test_start_session_timeout(client: AuthClient, start_url: str, code_challenge: str) -> None: responses.add(responses.POST, start_url, status=504) timeout_response = requests.post(start_url, timeout=5) @@ -77,14 +73,12 @@ def test_start_session_timeout(client: AuthClient, start_url: str, code_challeng responses.add(responses.POST, start_url, body=timeout_error) - with pytest.raises(CycodeError) as e_info: + with pytest.raises(RequestTimeoutError): client.start_session(code_challenge) - assert e_info.value.status_code == 504 - @responses.activate -def test_start_session_http_error(client: AuthClient, start_url: str, code_challenge: str): +def test_start_session_http_error(client: AuthClient, start_url: str, code_challenge: str) -> None: responses.add(responses.POST, start_url, status=401) with pytest.raises(CycodeError) as e_info: @@ -94,7 +88,7 @@ def test_start_session_http_error(client: AuthClient, start_url: str, code_chall @responses.activate -def test_get_api_token_success_pending(client: AuthClient, token_url: str, code_verifier: str): +def test_get_api_token_success_pending(client: AuthClient, token_url: str, code_verifier: str) -> None: expected_status = 'Pending' expected_api_token = None @@ -112,7 +106,7 @@ def test_get_api_token_success_pending(client: AuthClient, token_url: str, code_ @responses.activate -def test_get_api_token_success_completed(client: AuthClient, token_url: str, code_verifier: str): +def test_get_api_token_success_completed(client: AuthClient, token_url: str, code_verifier: str) -> None: expected_status = 'Completed' expected_json = { 'status': expected_status, @@ -121,8 +115,8 @@ def test_get_api_token_success_completed(client: AuthClient, token_url: str, cod 'secret': 'a123450a-42b2-4ad5-8bdd-c0130123456', 'description': 'cycode cli api token', 'createdByUserId': None, - 'createdAt': '2023-04-26T11:38:54+00:00' - } + 'createdAt': '2023-04-26T11:38:54+00:00', + }, } expected_response = ApiTokenGenerationPollingResponseSchema().load(expected_json) @@ -142,7 +136,7 @@ def test_get_api_token_success_completed(client: AuthClient, token_url: str, cod @responses.activate -def test_get_api_token_http_error_valid_response(client: AuthClient, token_url: str, code_verifier: str): +def test_get_api_token_http_error_valid_response(client: AuthClient, token_url: str, code_verifier: str) -> None: expected_status = 'Pending' expected_api_token = None @@ -160,7 +154,7 @@ def test_get_api_token_http_error_valid_response(client: AuthClient, token_url: @responses.activate -def test_get_api_token_http_error_invalid_response(client: AuthClient, token_url: str, code_verifier: str): +def test_get_api_token_http_error_invalid_response(client: AuthClient, token_url: str, code_verifier: str) -> None: responses.add( responses.POST, token_url, @@ -173,7 +167,7 @@ def test_get_api_token_http_error_invalid_response(client: AuthClient, token_url @responses.activate -def test_get_api_token_not_excepted_exception(client: AuthClient, token_url: str, code_verifier: str): +def test_get_api_token_not_excepted_exception(client: AuthClient, token_url: str, code_verifier: str) -> None: responses.add(responses.POST, token_url, body=Timeout()) api_token_polling_response = client.get_api_token(_SESSION_ID, code_verifier) diff --git a/tests/cyclient/test_client.py b/tests/cyclient/test_client.py index 623877a4..9c68cacf 100644 --- a/tests/cyclient/test_client.py +++ b/tests/cyclient/test_client.py @@ -2,7 +2,7 @@ from cycode.cyclient.cycode_client import CycodeClient -def test_init_values_from_config(): +def test_init_values_from_config() -> None: client = CycodeClient() assert client.api_url == config.cycode_api_url diff --git a/tests/cyclient/test_client_base.py b/tests/cyclient/test_client_base.py index 744ba097..d9e871d1 100644 --- a/tests/cyclient/test_client_base.py +++ b/tests/cyclient/test_client_base.py @@ -1,35 +1,35 @@ from cycode.cyclient import config -from cycode.cyclient.cycode_client_base import CycodeClientBase, get_cli_user_agent +from cycode.cyclient.cycode_client_base import CycodeClientBase +from cycode.cyclient.headers import get_cli_user_agent, get_correlation_id -def test_mandatory_headers(): +def test_mandatory_headers() -> None: expected_headers = { 'User-Agent': get_cli_user_agent(), + 'X-Correlation-Id': get_correlation_id(), } client = CycodeClientBase(config.cycode_api_url) - - assert client.MANDATORY_HEADERS == expected_headers + assert expected_headers == client.MANDATORY_HEADERS -def test_get_request_headers(): + +def test_get_request_headers() -> None: client = CycodeClientBase(config.cycode_api_url) assert client.get_request_headers() == client.MANDATORY_HEADERS -def test_get_request_headers_with_additional(): +def test_get_request_headers_with_additional() -> None: client = CycodeClientBase(config.cycode_api_url) - additional_headers = { - 'Authorize': 'Token test' - } + additional_headers = {'Authorize': 'Token test'} expected_headers = {**client.MANDATORY_HEADERS, **additional_headers} assert client.get_request_headers(additional_headers) == expected_headers -def test_build_full_url(): +def test_build_full_url() -> None: url = config.cycode_api_url client = CycodeClientBase(url) diff --git a/tests/cyclient/test_client_base_exceptions.py b/tests/cyclient/test_client_base_exceptions.py new file mode 100644 index 00000000..f99453d3 --- /dev/null +++ b/tests/cyclient/test_client_base_exceptions.py @@ -0,0 +1,162 @@ +from unittest.mock import MagicMock + +import pytest +import responses +from requests.exceptions import ( + ConnectionError as RequestsConnectionError, +) +from requests.exceptions import ( + HTTPError, + SSLError, + Timeout, +) + +from cycode.cli.exceptions.custom_exceptions import ( + HttpUnauthorizedError, + RequestConnectionError, + RequestHttpError, + RequestSslError, + RequestTimeoutError, +) +from cycode.cyclient import config +from cycode.cyclient.cycode_client_base import CycodeClientBase + + +def _make_client() -> CycodeClientBase: + return CycodeClientBase(config.cycode_api_url) + + +# --- _handle_exception mapping --- + + +def test_handle_exception_timeout() -> None: + client = _make_client() + with pytest.raises(RequestTimeoutError): + client._handle_exception(Timeout('timed out')) + + +def test_handle_exception_ssl_error() -> None: + client = _make_client() + with pytest.raises(RequestSslError): + client._handle_exception(SSLError('cert verify failed')) + + +def test_handle_exception_connection_error() -> None: + client = _make_client() + with pytest.raises(RequestConnectionError): + client._handle_exception(RequestsConnectionError('refused')) + + +def test_handle_exception_http_error_401() -> None: + response = MagicMock() + response.status_code = 401 + response.text = 'Unauthorized' + error = HTTPError(response=response) + + client = _make_client() + with pytest.raises(HttpUnauthorizedError): + client._handle_exception(error) + + +def test_handle_exception_http_error_500() -> None: + response = MagicMock() + response.status_code = 500 + response.text = 'Internal Server Error' + error = HTTPError(response=response) + + client = _make_client() + with pytest.raises(RequestHttpError) as exc_info: + client._handle_exception(error) + assert exc_info.value.status_code == 500 + + +def test_handle_exception_unknown_error_reraises() -> None: + client = _make_client() + with pytest.raises(RuntimeError, match='something unexpected'): + client._handle_exception(RuntimeError('something unexpected')) + + +# --- HTTP integration via responses mock --- + + +@responses.activate +def test_get_returns_response_on_success() -> None: + client = _make_client() + url = f'{client.api_url}/test-endpoint' + responses.add(responses.GET, url, json={'ok': True}, status=200) + + response = client.get('test-endpoint') + assert response.status_code == 200 + assert response.json() == {'ok': True} + + +@responses.activate +def test_post_returns_response_on_success() -> None: + client = _make_client() + url = f'{client.api_url}/test-endpoint' + responses.add(responses.POST, url, json={'created': True}, status=201) + + response = client.post('test-endpoint', body={'data': 'value'}) + assert response.status_code == 201 + + +@responses.activate +def test_get_raises_timeout_error() -> None: + client = _make_client() + url = f'{client.api_url}/slow-endpoint' + responses.add(responses.GET, url, body=Timeout('Connection timed out')) + + with pytest.raises(RequestTimeoutError): + client.get('slow-endpoint') + + +@responses.activate +def test_get_raises_ssl_error() -> None: + client = _make_client() + url = f'{client.api_url}/ssl-endpoint' + responses.add(responses.GET, url, body=SSLError('certificate verify failed')) + + with pytest.raises(RequestSslError): + client.get('ssl-endpoint') + + +@responses.activate +def test_get_raises_connection_error() -> None: + client = _make_client() + url = f'{client.api_url}/down-endpoint' + responses.add(responses.GET, url, body=RequestsConnectionError('Connection refused')) + + with pytest.raises(RequestConnectionError): + client.get('down-endpoint') + + +@responses.activate +def test_get_raises_http_unauthorized_error() -> None: + client = _make_client() + url = f'{client.api_url}/auth-endpoint' + responses.add(responses.GET, url, json={'error': 'unauthorized'}, status=401) + + with pytest.raises(HttpUnauthorizedError): + client.get('auth-endpoint') + + +@responses.activate +def test_get_raises_http_error_on_500() -> None: + client = _make_client() + url = f'{client.api_url}/error-endpoint' + responses.add(responses.GET, url, json={'error': 'server error'}, status=500) + + with pytest.raises(RequestHttpError) as exc_info: + client.get('error-endpoint') + assert exc_info.value.status_code == 500 + + +@responses.activate +def test_get_raises_http_error_on_403() -> None: + client = _make_client() + url = f'{client.api_url}/forbidden-endpoint' + responses.add(responses.GET, url, json={'error': 'forbidden'}, status=403) + + with pytest.raises(RequestHttpError) as exc_info: + client.get('forbidden-endpoint') + assert exc_info.value.status_code == 403 diff --git a/tests/cyclient/test_config.py b/tests/cyclient/test_config.py new file mode 100644 index 00000000..c78a5f5d --- /dev/null +++ b/tests/cyclient/test_config.py @@ -0,0 +1,24 @@ +from typing import TYPE_CHECKING + +from cycode.cli.consts import DEFAULT_CYCODE_DOMAIN +from cycode.cyclient.config import _is_on_premise_installation + +if TYPE_CHECKING: + from _pytest.monkeypatch import MonkeyPatch + + +def test_is_on_premise_installation(monkeypatch: 'MonkeyPatch') -> None: + monkeypatch.setattr('cycode.cyclient.config.cycode_api_url', 'api.cycode.com') + assert not _is_on_premise_installation(DEFAULT_CYCODE_DOMAIN) + monkeypatch.setattr('cycode.cyclient.config.cycode_api_url', 'api.eu.cycode.com') + assert not _is_on_premise_installation(DEFAULT_CYCODE_DOMAIN) + + monkeypatch.setattr('cycode.cyclient.config.cycode_api_url', 'cycode.google.com') + assert _is_on_premise_installation(DEFAULT_CYCODE_DOMAIN) + monkeypatch.setattr('cycode.cyclient.config.cycode_api_url', 'cycode.blabla.google.com') + assert _is_on_premise_installation(DEFAULT_CYCODE_DOMAIN) + + monkeypatch.setattr('cycode.cyclient.config.cycode_api_url', 'api.cycode.com') + assert _is_on_premise_installation('blabla') + monkeypatch.setattr('cycode.cyclient.config.cycode_api_url', 'cycode.blabla.google.com') + assert _is_on_premise_installation('blabla') diff --git a/tests/cyclient/test_dev_based_client.py b/tests/cyclient/test_dev_based_client.py index 6ce9f3be..f270761f 100644 --- a/tests/cyclient/test_dev_based_client.py +++ b/tests/cyclient/test_dev_based_client.py @@ -2,18 +2,16 @@ from cycode.cyclient.cycode_dev_based_client import CycodeDevBasedClient -def test_get_request_headers(): +def test_get_request_headers() -> None: client = CycodeDevBasedClient(config.cycode_api_url) - dev_based_headers = { - 'X-Tenant-Id': config.dev_tenant_id - } + dev_based_headers = {'X-Tenant-Id': config.dev_tenant_id} expected_headers = {**client.MANDATORY_HEADERS, **dev_based_headers} assert client.get_request_headers() == expected_headers -def test_build_full_url(): +def test_build_full_url() -> None: url = config.cycode_api_url client = CycodeDevBasedClient(url) diff --git a/tests/cyclient/test_oidc_based_client.py b/tests/cyclient/test_oidc_based_client.py new file mode 100644 index 00000000..070448b3 --- /dev/null +++ b/tests/cyclient/test_oidc_based_client.py @@ -0,0 +1,91 @@ +import arrow +import responses + +from cycode.cyclient.cycode_oidc_based_client import CycodeOidcBasedClient +from tests.conftest import _EXPECTED_API_TOKEN, create_oidc_based_client + + +@responses.activate +def test_access_token_new( + oidc_based_client: CycodeOidcBasedClient, oidc_api_token_response: responses.Response +) -> None: + responses.add(oidc_api_token_response) + + api_token = oidc_based_client.get_access_token() + + assert api_token == _EXPECTED_API_TOKEN + + +@responses.activate +def test_access_token_expired( + oidc_based_client: CycodeOidcBasedClient, oidc_api_token_response: responses.Response +) -> None: + responses.add(oidc_api_token_response) + + oidc_based_client.get_access_token() + + oidc_based_client._expires_in = arrow.utcnow().shift(hours=-1) + + api_token_refreshed = oidc_based_client.get_access_token() + + assert api_token_refreshed == _EXPECTED_API_TOKEN + + +def test_get_request_headers(oidc_based_client: CycodeOidcBasedClient, oidc_api_token: str) -> None: + expected_headers = { + **oidc_based_client.MANDATORY_HEADERS, + 'Authorization': f'Bearer {_EXPECTED_API_TOKEN}', + } + + assert oidc_based_client.get_request_headers() == expected_headers + + +@responses.activate +def test_access_token_cached( + oidc_based_client: CycodeOidcBasedClient, oidc_api_token_response: responses.Response +) -> None: + responses.add(oidc_api_token_response) + oidc_based_client.get_access_token() + + client2 = create_oidc_based_client() + assert client2._access_token == oidc_based_client._access_token + assert client2._expires_in == oidc_based_client._expires_in + + +@responses.activate +def test_access_token_cached_creator_changed( + oidc_based_client: CycodeOidcBasedClient, oidc_api_token_response: responses.Response +) -> None: + responses.add(oidc_api_token_response) + oidc_based_client.get_access_token() + + client2 = create_oidc_based_client('client_id2', 'different-token') + assert client2._access_token is None + assert client2._expires_in is None + + +@responses.activate +def test_access_token_invalidation( + oidc_based_client: CycodeOidcBasedClient, oidc_api_token_response: responses.Response +) -> None: + responses.add(oidc_api_token_response) + oidc_based_client.get_access_token() + + expected_access_token = oidc_based_client._access_token + expected_expires_in = oidc_based_client._expires_in + + oidc_based_client.invalidate_access_token() + assert oidc_based_client._access_token is None + assert oidc_based_client._expires_in is None + + client2 = create_oidc_based_client() + assert client2._access_token == expected_access_token + assert client2._expires_in == expected_expires_in + + client2.invalidate_access_token(in_storage=True) + assert client2._access_token is None + assert client2._expires_in is None + + client3 = create_oidc_based_client() + assert client3._access_token is None + assert client3._expires_in is None diff --git a/tests/cyclient/test_scan_client.py b/tests/cyclient/test_scan_client.py index afbbdabe..d6928118 100644 --- a/tests/cyclient/test_scan_client.py +++ b/tests/cyclient/test_scan_client.py @@ -1,186 +1,170 @@ import os -from uuid import uuid4, UUID +from uuid import uuid4 import pytest import requests import responses -from typing import List - -from requests import Timeout -from requests.exceptions import ProxyError - -from cycode.cli.config import config -from cycode.cli.zip_file import InMemoryZip +from requests.exceptions import ConnectionError as RequestsConnectionError + +from cycode.cli.cli_types import ScanTypeOption +from cycode.cli.exceptions.custom_exceptions import ( + CycodeError, + HttpUnauthorizedError, + RequestConnectionError, + RequestTimeoutError, +) +from cycode.cli.files_collector.models.in_memory_zip import InMemoryZip from cycode.cli.models import Document -from cycode.cli.code_scanner import zip_documents_to_scan -from cycode.cli.exceptions.custom_exceptions import HttpUnauthorizedError, CycodeError from cycode.cyclient.scan_client import ScanClient -from tests.conftest import TEST_FILES_PATH - - -_ZIP_CONTENT_PATH = TEST_FILES_PATH.joinpath('zip_content').absolute() - - -def zip_scan_resources(scan_type: str, scan_client: ScanClient): - url = get_zipped_file_scan_url(scan_type, scan_client) +from tests.conftest import ZIP_CONTENT_PATH +from tests.cyclient.mocked_responses.scan_client import ( + get_scan_aggregation_report_url, + get_scan_aggregation_report_url_response, + get_scan_details_response, + get_scan_details_url, + get_zipped_file_scan_async_response, + get_zipped_file_scan_async_url, +) + + +def zip_scan_resources(scan_type: ScanTypeOption, scan_client: ScanClient) -> tuple[str, InMemoryZip]: + url = get_zipped_file_scan_async_url(scan_type, scan_client) zip_file = get_test_zip_file(scan_type) return url, zip_file -def get_zipped_file_scan_url(scan_type: str, scan_client: ScanClient) -> str: - api_url = scan_client.scan_cycode_client.api_url - # TODO(MarshalX): create method in the scan client to build this url - return f'{api_url}/{scan_client.scan_config.get_service_name(scan_type)}/{scan_client.SCAN_CONTROLLER_PATH}/zipped-file' - - -def get_test_zip_file(scan_type: str) -> InMemoryZip: +def get_test_zip_file(scan_type: ScanTypeOption) -> InMemoryZip: # TODO(MarshalX): refactor scan_disk_files in code_scanner.py to reuse method here instead of this - test_documents: List[Document] = [] - for root, _, files in os.walk(_ZIP_CONTENT_PATH): + test_documents: list[Document] = [] + for root, _, files in os.walk(ZIP_CONTENT_PATH): for name in files: path = os.path.join(root, name) - with open(path, 'r', encoding='UTF-8') as f: + with open(path, encoding='UTF-8') as f: test_documents.append(Document(path, f.read(), is_git_diff_format=False)) - return zip_documents_to_scan(scan_type, InMemoryZip(), test_documents) - - -def get_zipped_file_scan_response(url: str, scan_id: UUID = None) -> responses.Response: - if not scan_id: - scan_id = uuid4() - - json_response = { - 'did_detect': True, - 'scan_id': scan_id.hex, # not always as expected due to _get_scan_id and passing scan_id to cxt of CLI - 'detections_per_file': [ - { - 'file_name': str(_ZIP_CONTENT_PATH.joinpath('secrets.py')), - 'commit_id': None, - 'detections': [ - { - 'detection_type_id': '12345678-418f-47ee-abb0-012345678901', - 'detection_rule_id': '12345678-aea1-4304-a6e9-012345678901', - 'message': "Secret of type 'Slack Token' was found in filename 'secrets.py'", - 'type': 'slack-token', - 'is_research': False, - 'detection_details': { - 'sha512': 'sha hash', - 'length': 55, - 'start_position': 19, - 'line': 0, - 'committed_at': '0001-01-01T00:00:00+00:00', - 'file_path': str(_ZIP_CONTENT_PATH), - 'file_name': 'secrets.py', - 'file_extension': '.py', - 'should_resolve_upon_branch_deletion': False - } - } - ] - } - ], - 'report_url': None - } - - return responses.Response(method=responses.POST, url=url, json=json_response, status=200) - - -def test_get_service_name(scan_client: ScanClient): - # TODO(Marshal): get_service_name should be removed from ScanClient? Because it exists in ScanConfig - assert scan_client.get_service_name('secret') == 'secret' - assert scan_client.get_service_name('iac') == 'iac' - assert scan_client.get_service_name('sca') == 'scans' - assert scan_client.get_service_name('sast') == 'scans' - - -@pytest.mark.parametrize('scan_type', config['scans']['supported_scans']) + from cycode.cli.files_collector.zip_documents import zip_documents + + return zip_documents(scan_type, test_documents) + + +@pytest.mark.parametrize('scan_type', list(ScanTypeOption)) @responses.activate -def test_zipped_file_scan(scan_type: str, scan_client: ScanClient, api_token_response): +def test_zipped_file_scan_async( + scan_type: ScanTypeOption, scan_client: ScanClient, api_token_response: responses.Response +) -> None: + """Test the zipped_file_scan_async method for the async flow.""" url, zip_file = zip_scan_resources(scan_type, scan_client) expected_scan_id = uuid4() - responses.add(api_token_response) # mock token based client - responses.add(get_zipped_file_scan_response(url, expected_scan_id)) + responses.add(api_token_response) # mock token based client + responses.add(get_zipped_file_scan_async_response(url, expected_scan_id)) + + scan_initialization_response = scan_client.zipped_file_scan_async(zip_file, scan_type, scan_parameters={}) + assert scan_initialization_response.scan_id == str(expected_scan_id) + - # TODO(MarshalX): fix wrong type hint? UUID instead of str - zipped_file_scan_response = scan_client.zipped_file_scan( - scan_type, zip_file, scan_id=expected_scan_id, scan_parameters={} - ) - assert zipped_file_scan_response.scan_id == expected_scan_id.hex +@pytest.mark.parametrize('scan_type', list(ScanTypeOption)) +@responses.activate +def test_get_scan_report_url( + scan_type: ScanTypeOption, scan_client: ScanClient, api_token_response: responses.Response +) -> None: + """Test getting the scan report URL for the async flow.""" + scan_id = uuid4() + url = get_scan_aggregation_report_url(scan_id, scan_client, scan_type) + + responses.add(api_token_response) # mock token based client + responses.add(get_scan_aggregation_report_url_response(url, scan_id)) + scan_report_url_response = scan_client.get_scan_aggregation_report_url(str(scan_id), scan_type) + assert scan_report_url_response.report_url == f'https://app.domain/cli-logs-aggregation/{scan_id}' -@pytest.mark.parametrize('scan_type', config['scans']['supported_scans']) + +@pytest.mark.parametrize('scan_type', list(ScanTypeOption)) @responses.activate -def test_zipped_file_scan_unauthorized_error(scan_type: str, scan_client: ScanClient, api_token_response): +def test_zipped_file_scan_async_unauthorized_error( + scan_type: ScanTypeOption, scan_client: ScanClient, api_token_response: responses.Response +) -> None: + """Test handling of unauthorized errors in the async flow.""" url, zip_file = zip_scan_resources(scan_type, scan_client) - expected_scan_id = uuid4().hex - responses.add(api_token_response) # mock token based client - responses.add(method=responses.POST, url=url, status=401) + responses.add(api_token_response) # mock token based client + responses.add(method=responses.POST, url=url, status=401, body='Unauthorized') with pytest.raises(HttpUnauthorizedError) as e_info: - scan_client.zipped_file_scan(scan_type, zip_file, scan_id=expected_scan_id, scan_parameters={}) + scan_client.zipped_file_scan_async(zip_file=zip_file, scan_type=scan_type, scan_parameters={}) assert e_info.value.status_code == 401 -@pytest.mark.parametrize('scan_type', config['scans']['supported_scans']) +@pytest.mark.parametrize('scan_type', list(ScanTypeOption)) @responses.activate -def test_zipped_file_scan_bad_request_error(scan_type: str, scan_client: ScanClient, api_token_response): +def test_zipped_file_scan_async_bad_request_error( + scan_type: ScanTypeOption, scan_client: ScanClient, api_token_response: responses.Response +) -> None: + """Test handling of bad request errors in the async flow.""" url, zip_file = zip_scan_resources(scan_type, scan_client) - expected_scan_id = uuid4().hex expected_status_code = 400 expected_response_text = 'Bad Request' - responses.add(api_token_response) # mock token based client + responses.add(api_token_response) # mock token based client responses.add(method=responses.POST, url=url, status=expected_status_code, body=expected_response_text) with pytest.raises(CycodeError) as e_info: - scan_client.zipped_file_scan(scan_type, zip_file, scan_id=expected_scan_id, scan_parameters={}) + scan_client.zipped_file_scan_async(zip_file=zip_file, scan_type=scan_type, scan_parameters={}) assert e_info.value.status_code == expected_status_code assert e_info.value.error_message == expected_response_text -@pytest.mark.parametrize('scan_type', config['scans']['supported_scans']) +@pytest.mark.parametrize('scan_type', list(ScanTypeOption)) @responses.activate -def test_zipped_file_scan_timeout_error(scan_type: str, scan_client: ScanClient, api_token_response): - scan_url, zip_file = zip_scan_resources(scan_type, scan_client) - expected_scan_id = uuid4().hex - - responses.add(responses.POST, scan_url, status=504) - - timeout_response = requests.post(scan_url, timeout=5) - if timeout_response.status_code == 504: - """bypass SAST""" - - responses.reset() - - timeout_error = Timeout() - timeout_error.response = timeout_response +def test_zipped_file_scan_async_timeout_error( + scan_type: ScanTypeOption, scan_client: ScanClient, api_token_response: responses.Response +) -> None: + """Test handling of timeout errors in the async flow.""" + url, zip_file = zip_scan_resources(scan_type, scan_client) - responses.add(api_token_response) # mock token based client - responses.add(method=responses.POST, url=scan_url, body=timeout_error, status=504) + timeout_error = requests.exceptions.Timeout('Connection timed out') - with pytest.raises(CycodeError) as e_info: - scan_client.zipped_file_scan(scan_type, zip_file, scan_id=expected_scan_id, scan_parameters={}) + responses.add(api_token_response) # mock token based client + responses.add(method=responses.POST, url=url, body=timeout_error) - assert e_info.value.status_code == 504 - assert e_info.value.error_message == 'Timeout Error' + with pytest.raises(RequestTimeoutError): + scan_client.zipped_file_scan_async(zip_file=zip_file, scan_type=scan_type, scan_parameters={}) -@pytest.mark.parametrize('scan_type', config['scans']['supported_scans']) +@pytest.mark.parametrize('scan_type', list(ScanTypeOption)) @responses.activate -def test_zipped_file_scan_connection_error(scan_type: str, scan_client: ScanClient, api_token_response): +def test_zipped_file_scan_async_connection_error( + scan_type: ScanTypeOption, scan_client: ScanClient, api_token_response: responses.Response +) -> None: + """Test handling of connection errors in the async flow.""" url, zip_file = zip_scan_resources(scan_type, scan_client) - expected_scan_id = uuid4().hex - responses.add(api_token_response) # mock token based client - responses.add(method=responses.POST, url=url, body=ProxyError()) + # Create a connection error response + connection_error = RequestsConnectionError('Connection refused') - with pytest.raises(CycodeError) as e_info: - scan_client.zipped_file_scan(scan_type, zip_file, scan_id=expected_scan_id, scan_parameters={}) + responses.add(api_token_response) # mock token based client + responses.add(method=responses.POST, url=url, body=connection_error) - assert e_info.value.status_code == 502 - assert e_info.value.error_message == 'Connection Error' + with pytest.raises(RequestConnectionError): + scan_client.zipped_file_scan_async(zip_file=zip_file, scan_type=scan_type, scan_parameters={}) + + +@pytest.mark.parametrize('scan_type', list(ScanTypeOption)) +@responses.activate +def test_get_scan_details( + scan_type: ScanTypeOption, scan_client: ScanClient, api_token_response: responses.Response +) -> None: + """Test getting scan details in the async flow.""" + scan_id = uuid4() + url = get_scan_details_url(scan_type, scan_id, scan_client) + + responses.add(api_token_response) # mock token based client + responses.add(get_scan_details_response(url, scan_id)) + + scan_details_response = scan_client.get_scan_details(scan_type, str(scan_id)) + assert scan_details_response.id == str(scan_id) + assert scan_details_response.scan_status == 'Completed' diff --git a/tests/cyclient/test_token_based_client.py b/tests/cyclient/test_token_based_client.py index 1ffa8948..4c3dd4c5 100644 --- a/tests/cyclient/test_token_based_client.py +++ b/tests/cyclient/test_token_based_client.py @@ -2,38 +2,97 @@ import responses from cycode.cyclient.cycode_token_based_client import CycodeTokenBasedClient -from ..conftest import _EXPECTED_API_TOKEN +from tests.conftest import _EXPECTED_API_TOKEN, create_token_based_client @responses.activate -def test_api_token_new(token_based_client: CycodeTokenBasedClient, api_token_response: responses.Response): +def test_access_token_new(token_based_client: CycodeTokenBasedClient, api_token_response: responses.Response) -> None: responses.add(api_token_response) - api_token = token_based_client.api_token + api_token = token_based_client.get_access_token() assert api_token == _EXPECTED_API_TOKEN @responses.activate -def test_api_token_expired(token_based_client: CycodeTokenBasedClient, api_token_response: responses.Response): +def test_access_token_expired( + token_based_client: CycodeTokenBasedClient, api_token_response: responses.Response +) -> None: responses.add(api_token_response) - # this property performs HTTP req to refresh the token. IDE doesn't know it - token_based_client.api_token + token_based_client.get_access_token() # mark token as expired token_based_client._expires_in = arrow.utcnow().shift(hours=-1) # refresh token - api_token_refreshed = token_based_client.api_token + api_token_refreshed = token_based_client.get_access_token() assert api_token_refreshed == _EXPECTED_API_TOKEN -def test_get_request_headers(token_based_client: CycodeTokenBasedClient, api_token: str): - token_based_headers = { - 'Authorization': f'Bearer {_EXPECTED_API_TOKEN}' - } +def test_get_request_headers(token_based_client: CycodeTokenBasedClient, api_token: str) -> None: + token_based_headers = {'Authorization': f'Bearer {_EXPECTED_API_TOKEN}'} expected_headers = {**token_based_client.MANDATORY_HEADERS, **token_based_headers} assert token_based_client.get_request_headers() == expected_headers + + +@responses.activate +def test_access_token_cached( + token_based_client: CycodeTokenBasedClient, api_token_response: responses.Response +) -> None: + # save to cache + responses.add(api_token_response) + token_based_client.get_access_token() + + # load from cache + client2 = create_token_based_client() + assert client2._access_token == token_based_client._access_token + assert client2._expires_in == token_based_client._expires_in + + +@responses.activate +def test_access_token_cached_creator_changed( + token_based_client: CycodeTokenBasedClient, api_token_response: responses.Response +) -> None: + # save to cache + responses.add(api_token_response) + token_based_client.get_access_token() + + # load from cache with another client id and client secret + client2 = create_token_based_client('client_id2', 'client_secret2') + assert client2._access_token is None + assert client2._expires_in is None + + +@responses.activate +def test_access_token_invalidation( + token_based_client: CycodeTokenBasedClient, api_token_response: responses.Response +) -> None: + # save to cache + responses.add(api_token_response) + token_based_client.get_access_token() + + expected_access_token = token_based_client._access_token + expected_expires_in = token_based_client._expires_in + + # invalidate in runtime + token_based_client.invalidate_access_token() + assert token_based_client._access_token is None + assert token_based_client._expires_in is None + + # load from cache + client2 = create_token_based_client() + assert client2._access_token == expected_access_token + assert client2._expires_in == expected_expires_in + + # invalidate in storage + client2.invalidate_access_token(in_storage=True) + assert client2._access_token is None + assert client2._expires_in is None + + # load from cache again + client3 = create_token_based_client() + assert client3._access_token is None + assert client3._expires_in is None diff --git a/tests/test_aggregation_report.py b/tests/test_aggregation_report.py new file mode 100644 index 00000000..b09c4d69 --- /dev/null +++ b/tests/test_aggregation_report.py @@ -0,0 +1,57 @@ +import os +from uuid import uuid4 + +import pytest +import responses + +from cycode.cli import consts +from cycode.cli.apps.scan.aggregation_report import try_get_aggregation_report_url_if_needed +from cycode.cli.cli_types import ScanTypeOption +from cycode.cli.files_collector.file_excluder import excluder +from cycode.cyclient.scan_client import ScanClient +from tests.conftest import TEST_FILES_PATH +from tests.cyclient.mocked_responses.scan_client import ( + get_scan_aggregation_report_url, + get_scan_aggregation_report_url_response, +) + + +def test_is_relevant_file_to_scan_sca() -> None: + path = os.path.join(TEST_FILES_PATH, 'package.json') + assert excluder._is_relevant_file_to_scan(consts.SCA_SCAN_TYPE, path) is True + + +@pytest.mark.parametrize('scan_type', list(ScanTypeOption)) +def test_try_get_aggregation_report_url_if_no_report_command_needed_return_none( + scan_type: ScanTypeOption, scan_client: ScanClient +) -> None: + aggregation_id = uuid4().hex + scan_parameter = {'aggregation_id': aggregation_id} + result = try_get_aggregation_report_url_if_needed(scan_parameter, scan_client, scan_type) + assert result is None + + +@pytest.mark.parametrize('scan_type', list(ScanTypeOption)) +def test_try_get_aggregation_report_url_if_no_aggregation_id_needed_return_none( + scan_type: ScanTypeOption, scan_client: ScanClient +) -> None: + scan_parameter = {'report': True} + result = try_get_aggregation_report_url_if_needed(scan_parameter, scan_client, scan_type) + assert result is None + + +@pytest.mark.parametrize('scan_type', list(ScanTypeOption)) +@responses.activate +def test_try_get_aggregation_report_url_if_needed_return_result( + scan_type: ScanTypeOption, scan_client: ScanClient, api_token_response: responses.Response +) -> None: + aggregation_id = uuid4() + scan_parameter = {'report': True, 'aggregation_id': aggregation_id} + url = get_scan_aggregation_report_url(aggregation_id, scan_client, scan_type) + responses.add(api_token_response) # mock token based client + responses.add(get_scan_aggregation_report_url_response(url, aggregation_id)) + + scan_aggregation_report_url_response = scan_client.get_scan_aggregation_report_url(str(aggregation_id), scan_type) + + result = try_get_aggregation_report_url_if_needed(scan_parameter, scan_client, scan_type) + assert result == scan_aggregation_report_url_response.report_url diff --git a/tests/test_code_scanner.py b/tests/test_code_scanner.py deleted file mode 100644 index 9e17a0c5..00000000 --- a/tests/test_code_scanner.py +++ /dev/null @@ -1,9 +0,0 @@ -import os - -from cycode.cli import code_scanner -from tests.conftest import TEST_FILES_PATH - - -def test_is_relevant_file_to_scan_sca(): - path = os.path.join(TEST_FILES_PATH, 'package.json') - assert code_scanner._is_relevant_file_to_scan('sca', path) is True diff --git a/tests/test_files/.test_env b/tests/test_files/.test_env new file mode 100644 index 00000000..3e58295a --- /dev/null +++ b/tests/test_files/.test_env @@ -0,0 +1 @@ +TELEGRAM_BOT_TOKEN=923445010:AAGWKwWTNx_6RAuRdcp2kWax5_JltwkF2Lw diff --git a/tests/test_files/tf_content_generator_files/tfplan-create-example/tf_content.txt b/tests/test_files/tf_content_generator_files/tfplan-create-example/tf_content.txt new file mode 100644 index 00000000..1a1761b8 --- /dev/null +++ b/tests/test_files/tf_content_generator_files/tfplan-create-example/tf_content.txt @@ -0,0 +1,113 @@ +resource "aws_codebuild_project" "terra_ci" { + artifacts = [{"artifact_identifier": null, "encryption_disabled": false, "location": "terra-ci-artifacts-eu-west-1-000002", "name": null, "namespace_type": null, "override_artifact_name": false, "packaging": null, "path": null, "type": "S3"}] + badge_enabled = false + build_timeout = 10 + cache = [] + description = "Deploy environment configuration" + environment = [{"certificate": null, "compute_type": "BUILD_GENERAL1_SMALL", "environment_variable": [], "image": "aws/codebuild/amazonlinux2-x86_64-standard:2.0", "image_pull_credentials_type": "CODEBUILD", "privileged_mode": false, "registry_credential": [], "type": "LINUX_CONTAINER"}] + logs_config = [{"cloudwatch_logs": [{"group_name": null, "status": "ENABLED", "stream_name": null}], "s3_logs": [{"encryption_disabled": false, "location": null, "status": "DISABLED"}]}] + name = "terra-ci-runner" + queued_timeout = 480 + secondary_artifacts = [] + secondary_sources = [] + source = [{"auth": [], "buildspec": "version: 0.2\nphases:\n install:\n commands:\n - make install_tools\n build:\n commands:\n - make plan_local resource=$TERRA_CI_RESOURCE\nartifacts:\n files:\n - ./tfplan\n name: $TERRA_CI_BUILD_NAME\n\n", "git_clone_depth": 1, "git_submodules_config": [], "insecure_ssl": false, "location": "https://github.com/p0tr3c-terraform/terra-ci-single-account.git", "report_build_status": false, "type": "GITHUB"}] + source_version = null + tags = null + vpc_config = [] +} + +resource "aws_iam_role" "terra_ci_job" { + assume_role_policy = "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Principal\": {\n \"Service\": \"codebuild.amazonaws.com\"\n },\n \"Action\": \"sts:AssumeRole\"\n }\n ]\n}\n" + description = null + force_detach_policies = false + max_session_duration = 3600 + name = "terra_ci_job" + name_prefix = null + path = "/" + permissions_boundary = null + tags = null +} + +resource "aws_iam_role" "terra_ci_runner" { + assume_role_policy = "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Principal\": {\n \"Service\": \"states.amazonaws.com\"\n },\n \"Action\": \"sts:AssumeRole\"\n }\n ]\n}\n" + description = null + force_detach_policies = false + max_session_duration = 3600 + name = "terra_ci_runner" + name_prefix = null + path = "/" + permissions_boundary = null + tags = null +} + +resource "aws_iam_role_policy" "terra_ci_job" { + name_prefix = null + role = "terra_ci_job" +} + +resource "aws_iam_role_policy" "terra_ci_runner" { + name_prefix = null + role = "terra_ci_runner" +} + +resource "aws_iam_role_policy_attachment" "terra_ci_job_ecr_access" { + policy_arn = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryPowerUser" + role = "terra_ci_job" +} + +resource "aws_s3_bucket" "terra_ci" { + acl = "private" + bucket = "terra-ci-artifacts-eu-west-1-000002" + bucket_prefix = null + cors_rule = [] + force_destroy = false + grant = [] + lifecycle_rule = [] + logging = [] + object_lock_configuration = [] + policy = null + replication_configuration = [] + server_side_encryption_configuration = [{"rule": [{"apply_server_side_encryption_by_default": [{"kms_master_key_id": null, "sse_algorithm": "aws:kms"}], "bucket_key_enabled": false}]}] + tags = null + website = [] +} + +resource "aws_sfn_state_machine" "terra_ci_runner" { + definition = "{\n \"Comment\": \"Run Terragrunt Jobs\",\n \"StartAt\": \"OnBranch?\",\n \"States\": {\n \"OnBranch?\": {\n \"Type\": \"Choice\",\n \"Choices\": [\n {\n \"Variable\": \"$.build.sourceversion\",\n \"IsPresent\": true,\n \"Next\": \"PlanBranch\"\n }\n ],\n \"Default\": \"Plan\"\n },\n \"Plan\": {\n \"Type\": \"Task\",\n \"Resource\": \"arn:aws:states:::codebuild:startBuild.sync\",\n \"Parameters\": {\n \"ProjectName\": \"terra-ci-runner\",\n \"EnvironmentVariablesOverride\": [\n {\n \"Name\": \"TERRA_CI_BUILD_NAME\",\n \"Value.$\": \"$$.Execution.Name\"\n },\n {\n \"Name\": \"TERRA_CI_RESOURCE\",\n \"Value.$\": \"$.build.environment.terra_ci_resource\"\n }\n ]\n },\n \"End\": true\n },\n \"PlanBranch\": {\n \"Type\": \"Task\",\n \"Resource\": \"arn:aws:states:::codebuild:startBuild.sync\",\n \"Parameters\": {\n \"ProjectName\": \"terra-ci-runner\",\n \"SourceVersion.$\": \"$.build.sourceversion\",\n \"EnvironmentVariablesOverride\": [\n {\n \"Name\": \"TERRA_CI_RESOURCE\",\n \"Value.$\": \"$.build.environment.terra_ci_resource\"\n }\n ]\n },\n \"End\": true\n }\n }\n}\n" + name = "terra-ci-runner" + tags = null + type = "STANDARD" +} + +resource "aws_route" "private.1" { + carrier_gateway_id = null + destination_cidr_block = "172.25.16.0/20" + destination_ipv6_cidr_block = null + destination_prefix_list_id = null + egress_only_gateway_id = null + gateway_id = null + local_gateway_id = null + nat_gateway_id = null + route_table_id = "rtb-00cf8381520103cfb" + timeouts = null + transit_gateway_id = "tgw-0f68a4f2c58772c51" + vpc_endpoint_id = null + vpc_peering_connection_id = null +} + +resource "aws_route" "private.rtb-00cf8381520103cfb" { + carrier_gateway_id = null + destination_cidr_block = "172.25.16.0/20" + destination_ipv6_cidr_block = null + destination_prefix_list_id = null + egress_only_gateway_id = null + gateway_id = null + local_gateway_id = null + nat_gateway_id = null + route_table_id = "rtb-00cf8381520103cfb" + timeouts = null + transit_gateway_id = "tgw-0f68a4f2c58772c51" + vpc_endpoint_id = null + vpc_peering_connection_id = null +} + diff --git a/tests/test_files/tf_content_generator_files/tfplan-create-example/tfplan.json b/tests/test_files/tf_content_generator_files/tfplan-create-example/tfplan.json new file mode 100644 index 00000000..0b3a9296 --- /dev/null +++ b/tests/test_files/tf_content_generator_files/tfplan-create-example/tfplan.json @@ -0,0 +1,1072 @@ +{ + "comment-for-reader": "THIS TERRAFORM FILE REPRESENTS A REAL TF-PLAN OUTPUT FOR NEWLY CREATED INFRASTRUCTRE", + "format_version": "0.1", + "terraform_version": "0.15.0", + "variables": { + "aws_region": { + "value": "eu-west-1" + }, + "repo_url": { + "value": "https://github.com/p0tr3c-terraform/terra-ci-single-account.git" + }, + "serial_number": { + "value": "000002" + } + }, + "planned_values": { + "root_module": { + "resources": [ + { + "address": "aws_codebuild_project.terra_ci", + "mode": "managed", + "type": "aws_codebuild_project", + "name": "terra_ci", + "provider_name": "registry.terraform.io/hashicorp/aws", + "schema_version": 0, + "values": { + "artifacts": [ + { + "artifact_identifier": null, + "encryption_disabled": false, + "location": "terra-ci-artifacts-eu-west-1-000002", + "name": null, + "namespace_type": null, + "override_artifact_name": false, + "packaging": null, + "path": null, + "type": "S3" + } + ], + "badge_enabled": false, + "build_timeout": 10, + "cache": [], + "description": "Deploy environment configuration", + "environment": [ + { + "certificate": null, + "compute_type": "BUILD_GENERAL1_SMALL", + "environment_variable": [], + "image": "aws/codebuild/amazonlinux2-x86_64-standard:2.0", + "image_pull_credentials_type": "CODEBUILD", + "privileged_mode": false, + "registry_credential": [], + "type": "LINUX_CONTAINER" + } + ], + "logs_config": [ + { + "cloudwatch_logs": [ + { + "group_name": null, + "status": "ENABLED", + "stream_name": null + } + ], + "s3_logs": [ + { + "encryption_disabled": false, + "location": null, + "status": "DISABLED" + } + ] + } + ], + "name": "terra-ci-runner", + "queued_timeout": 480, + "secondary_artifacts": [], + "secondary_sources": [], + "source": [ + { + "auth": [], + "buildspec": "version: 0.2\nphases:\n install:\n commands:\n - make install_tools\n build:\n commands:\n - make plan_local resource=$TERRA_CI_RESOURCE\nartifacts:\n files:\n - ./tfplan\n name: $TERRA_CI_BUILD_NAME\n\n", + "git_clone_depth": 1, + "git_submodules_config": [], + "insecure_ssl": false, + "location": "https://github.com/p0tr3c-terraform/terra-ci-single-account.git", + "report_build_status": false, + "type": "GITHUB" + } + ], + "source_version": null, + "tags": null, + "vpc_config": [] + } + }, + { + "address": "aws_iam_role.terra_ci_job", + "mode": "managed", + "type": "aws_iam_role", + "name": "terra_ci_job", + "provider_name": "registry.terraform.io/hashicorp/aws", + "schema_version": 0, + "values": { + "assume_role_policy": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Principal\": {\n \"Service\": \"codebuild.amazonaws.com\"\n },\n \"Action\": \"sts:AssumeRole\"\n }\n ]\n}\n", + "description": null, + "force_detach_policies": false, + "max_session_duration": 3600, + "name": "terra_ci_job", + "name_prefix": null, + "path": "/", + "permissions_boundary": null, + "tags": null + } + }, + { + "address": "aws_iam_role.terra_ci_runner", + "mode": "managed", + "type": "aws_iam_role", + "name": "terra_ci_runner", + "provider_name": "registry.terraform.io/hashicorp/aws", + "schema_version": 0, + "values": { + "assume_role_policy": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Principal\": {\n \"Service\": \"states.amazonaws.com\"\n },\n \"Action\": \"sts:AssumeRole\"\n }\n ]\n}\n", + "description": null, + "force_detach_policies": false, + "max_session_duration": 3600, + "name": "terra_ci_runner", + "name_prefix": null, + "path": "/", + "permissions_boundary": null, + "tags": null + } + }, + { + "address": "aws_iam_role_policy.terra_ci_job", + "mode": "managed", + "type": "aws_iam_role_policy", + "name": "terra_ci_job", + "provider_name": "registry.terraform.io/hashicorp/aws", + "schema_version": 0, + "values": { + "name_prefix": null, + "role": "terra_ci_job" + } + }, + { + "address": "aws_iam_role_policy.terra_ci_runner", + "mode": "managed", + "type": "aws_iam_role_policy", + "name": "terra_ci_runner", + "provider_name": "registry.terraform.io/hashicorp/aws", + "schema_version": 0, + "values": { + "name_prefix": null, + "role": "terra_ci_runner" + } + }, + { + "address": "aws_iam_role_policy_attachment.terra_ci_job_ecr_access", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "terra_ci_job_ecr_access", + "provider_name": "registry.terraform.io/hashicorp/aws", + "schema_version": 0, + "values": { + "policy_arn": "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryPowerUser", + "role": "terra_ci_job" + } + }, + { + "address": "aws_s3_bucket.terra_ci", + "mode": "managed", + "type": "aws_s3_bucket", + "name": "terra_ci", + "provider_name": "registry.terraform.io/hashicorp/aws", + "schema_version": 0, + "values": { + "acl": "private", + "bucket": "terra-ci-artifacts-eu-west-1-000002", + "bucket_prefix": null, + "cors_rule": [], + "force_destroy": false, + "grant": [], + "lifecycle_rule": [], + "logging": [], + "object_lock_configuration": [], + "policy": null, + "replication_configuration": [], + "server_side_encryption_configuration": [ + { + "rule": [ + { + "apply_server_side_encryption_by_default": [ + { + "kms_master_key_id": null, + "sse_algorithm": "aws:kms" + } + ], + "bucket_key_enabled": false + } + ] + } + ], + "tags": null, + "website": [] + } + }, + { + "address": "aws_sfn_state_machine.terra_ci_runner", + "mode": "managed", + "type": "aws_sfn_state_machine", + "name": "terra_ci_runner", + "provider_name": "registry.terraform.io/hashicorp/aws", + "schema_version": 0, + "values": { + "definition": "{\n \"Comment\": \"Run Terragrunt Jobs\",\n \"StartAt\": \"OnBranch?\",\n \"States\": {\n \"OnBranch?\": {\n \"Type\": \"Choice\",\n \"Choices\": [\n {\n \"Variable\": \"$.build.sourceversion\",\n \"IsPresent\": true,\n \"Next\": \"PlanBranch\"\n }\n ],\n \"Default\": \"Plan\"\n },\n \"Plan\": {\n \"Type\": \"Task\",\n \"Resource\": \"arn:aws:states:::codebuild:startBuild.sync\",\n \"Parameters\": {\n \"ProjectName\": \"terra-ci-runner\",\n \"EnvironmentVariablesOverride\": [\n {\n \"Name\": \"TERRA_CI_BUILD_NAME\",\n \"Value.$\": \"$$.Execution.Name\"\n },\n {\n \"Name\": \"TERRA_CI_RESOURCE\",\n \"Value.$\": \"$.build.environment.terra_ci_resource\"\n }\n ]\n },\n \"End\": true\n },\n \"PlanBranch\": {\n \"Type\": \"Task\",\n \"Resource\": \"arn:aws:states:::codebuild:startBuild.sync\",\n \"Parameters\": {\n \"ProjectName\": \"terra-ci-runner\",\n \"SourceVersion.$\": \"$.build.sourceversion\",\n \"EnvironmentVariablesOverride\": [\n {\n \"Name\": \"TERRA_CI_RESOURCE\",\n \"Value.$\": \"$.build.environment.terra_ci_resource\"\n }\n ]\n },\n \"End\": true\n }\n }\n}\n", + "name": "terra-ci-runner", + "tags": null, + "type": "STANDARD" + } + } + ] + } + }, + "resource_changes": [ + { + "address": "aws_codebuild_project.terra_ci", + "mode": "managed", + "type": "aws_codebuild_project", + "name": "terra_ci", + "provider_name": "registry.terraform.io/hashicorp/aws", + "change": { + "actions": [ + "create" + ], + "before": null, + "after": { + "artifacts": [ + { + "artifact_identifier": null, + "encryption_disabled": false, + "location": "terra-ci-artifacts-eu-west-1-000002", + "name": null, + "namespace_type": null, + "override_artifact_name": false, + "packaging": null, + "path": null, + "type": "S3" + } + ], + "badge_enabled": false, + "build_timeout": 10, + "cache": [], + "description": "Deploy environment configuration", + "environment": [ + { + "certificate": null, + "compute_type": "BUILD_GENERAL1_SMALL", + "environment_variable": [], + "image": "aws/codebuild/amazonlinux2-x86_64-standard:2.0", + "image_pull_credentials_type": "CODEBUILD", + "privileged_mode": false, + "registry_credential": [], + "type": "LINUX_CONTAINER" + } + ], + "logs_config": [ + { + "cloudwatch_logs": [ + { + "group_name": null, + "status": "ENABLED", + "stream_name": null + } + ], + "s3_logs": [ + { + "encryption_disabled": false, + "location": null, + "status": "DISABLED" + } + ] + } + ], + "name": "terra-ci-runner", + "queued_timeout": 480, + "secondary_artifacts": [], + "secondary_sources": [], + "source": [ + { + "auth": [], + "buildspec": "version: 0.2\nphases:\n install:\n commands:\n - make install_tools\n build:\n commands:\n - make plan_local resource=$TERRA_CI_RESOURCE\nartifacts:\n files:\n - ./tfplan\n name: $TERRA_CI_BUILD_NAME\n\n", + "git_clone_depth": 1, + "git_submodules_config": [], + "insecure_ssl": false, + "location": "https://github.com/p0tr3c-terraform/terra-ci-single-account.git", + "report_build_status": false, + "type": "GITHUB" + } + ], + "source_version": null, + "tags": null, + "vpc_config": [] + }, + "after_unknown": { + "arn": true, + "artifacts": [ + {} + ], + "badge_url": true, + "cache": [], + "encryption_key": true, + "environment": [ + { + "environment_variable": [], + "registry_credential": [] + } + ], + "id": true, + "logs_config": [ + { + "cloudwatch_logs": [ + {} + ], + "s3_logs": [ + {} + ] + } + ], + "secondary_artifacts": [], + "secondary_sources": [], + "service_role": true, + "source": [ + { + "auth": [], + "git_submodules_config": [] + } + ], + "vpc_config": [] + }, + "before_sensitive": false, + "after_sensitive": { + "artifacts": [ + {} + ], + "cache": [], + "environment": [ + { + "environment_variable": [], + "registry_credential": [] + } + ], + "logs_config": [ + { + "cloudwatch_logs": [ + {} + ], + "s3_logs": [ + {} + ] + } + ], + "secondary_artifacts": [], + "secondary_sources": [], + "source": [ + { + "auth": [], + "git_submodules_config": [] + } + ], + "vpc_config": [] + } + } + }, + { + "address": "aws_iam_role.terra_ci_job", + "mode": "managed", + "type": "aws_iam_role", + "name": "terra_ci_job", + "provider_name": "registry.terraform.io/hashicorp/aws", + "change": { + "actions": [ + "create" + ], + "before": null, + "after": { + "assume_role_policy": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Principal\": {\n \"Service\": \"codebuild.amazonaws.com\"\n },\n \"Action\": \"sts:AssumeRole\"\n }\n ]\n}\n", + "description": null, + "force_detach_policies": false, + "max_session_duration": 3600, + "name": "terra_ci_job", + "name_prefix": null, + "path": "/", + "permissions_boundary": null, + "tags": null + }, + "after_unknown": { + "arn": true, + "create_date": true, + "id": true, + "inline_policy": true, + "managed_policy_arns": true, + "unique_id": true + }, + "before_sensitive": false, + "after_sensitive": { + "inline_policy": [], + "managed_policy_arns": [] + } + } + }, + { + "address": "aws_iam_role.terra_ci_runner", + "mode": "managed", + "type": "aws_iam_role", + "name": "terra_ci_runner", + "provider_name": "registry.terraform.io/hashicorp/aws", + "change": { + "actions": [ + "create" + ], + "before": null, + "after": { + "assume_role_policy": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Principal\": {\n \"Service\": \"states.amazonaws.com\"\n },\n \"Action\": \"sts:AssumeRole\"\n }\n ]\n}\n", + "description": null, + "force_detach_policies": false, + "max_session_duration": 3600, + "name": "terra_ci_runner", + "name_prefix": null, + "path": "/", + "permissions_boundary": null, + "tags": null + }, + "after_unknown": { + "arn": true, + "create_date": true, + "id": true, + "inline_policy": true, + "managed_policy_arns": true, + "unique_id": true + }, + "before_sensitive": false, + "after_sensitive": { + "inline_policy": [], + "managed_policy_arns": [] + } + } + }, + { + "address": "aws_iam_role_policy.terra_ci_job", + "mode": "managed", + "type": "aws_iam_role_policy", + "name": "terra_ci_job", + "provider_name": "registry.terraform.io/hashicorp/aws", + "change": { + "actions": [ + "create" + ], + "before": null, + "after": { + "name_prefix": null, + "role": "terra_ci_job" + }, + "after_unknown": { + "id": true, + "name": true, + "policy": true + }, + "before_sensitive": false, + "after_sensitive": {} + } + }, + { + "address": "aws_iam_role_policy.terra_ci_runner", + "mode": "managed", + "type": "aws_iam_role_policy", + "name": "terra_ci_runner", + "provider_name": "registry.terraform.io/hashicorp/aws", + "change": { + "actions": [ + "create" + ], + "before": null, + "after": { + "name_prefix": null, + "role": "terra_ci_runner" + }, + "after_unknown": { + "id": true, + "name": true, + "policy": true + }, + "before_sensitive": false, + "after_sensitive": {} + } + }, + { + "address": "aws_iam_role_policy_attachment.terra_ci_job_ecr_access", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "terra_ci_job_ecr_access", + "provider_name": "registry.terraform.io/hashicorp/aws", + "change": { + "actions": [ + "create" + ], + "before": null, + "after": { + "policy_arn": "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryPowerUser", + "role": "terra_ci_job" + }, + "after_unknown": { + "id": true + }, + "before_sensitive": false, + "after_sensitive": {} + } + }, + { + "address": "aws_s3_bucket.terra_ci", + "mode": "managed", + "type": "aws_s3_bucket", + "name": "terra_ci", + "provider_name": "registry.terraform.io/hashicorp/aws", + "change": { + "actions": [ + "create" + ], + "before": null, + "after": { + "acl": "private", + "bucket": "terra-ci-artifacts-eu-west-1-000002", + "bucket_prefix": null, + "cors_rule": [], + "force_destroy": false, + "grant": [], + "lifecycle_rule": [], + "logging": [], + "object_lock_configuration": [], + "policy": null, + "replication_configuration": [], + "server_side_encryption_configuration": [ + { + "rule": [ + { + "apply_server_side_encryption_by_default": [ + { + "kms_master_key_id": null, + "sse_algorithm": "aws:kms" + } + ], + "bucket_key_enabled": false + } + ] + } + ], + "tags": null, + "website": [] + }, + "after_unknown": { + "acceleration_status": true, + "arn": true, + "bucket_domain_name": true, + "bucket_regional_domain_name": true, + "cors_rule": [], + "grant": [], + "hosted_zone_id": true, + "id": true, + "lifecycle_rule": [], + "logging": [], + "object_lock_configuration": [], + "region": true, + "replication_configuration": [], + "request_payer": true, + "server_side_encryption_configuration": [ + { + "rule": [ + { + "apply_server_side_encryption_by_default": [ + {} + ] + } + ] + } + ], + "versioning": true, + "website": [], + "website_domain": true, + "website_endpoint": true + }, + "before_sensitive": false, + "after_sensitive": { + "cors_rule": [], + "grant": [], + "lifecycle_rule": [], + "logging": [], + "object_lock_configuration": [], + "replication_configuration": [], + "server_side_encryption_configuration": [ + { + "rule": [ + { + "apply_server_side_encryption_by_default": [ + {} + ] + } + ] + } + ], + "versioning": [], + "website": [] + } + } + }, + { + "address": "aws_sfn_state_machine.terra_ci_runner", + "mode": "managed", + "type": "aws_sfn_state_machine", + "name": "terra_ci_runner", + "provider_name": "registry.terraform.io/hashicorp/aws", + "change": { + "actions": [ + "create" + ], + "before": null, + "after": { + "definition": "{\n \"Comment\": \"Run Terragrunt Jobs\",\n \"StartAt\": \"OnBranch?\",\n \"States\": {\n \"OnBranch?\": {\n \"Type\": \"Choice\",\n \"Choices\": [\n {\n \"Variable\": \"$.build.sourceversion\",\n \"IsPresent\": true,\n \"Next\": \"PlanBranch\"\n }\n ],\n \"Default\": \"Plan\"\n },\n \"Plan\": {\n \"Type\": \"Task\",\n \"Resource\": \"arn:aws:states:::codebuild:startBuild.sync\",\n \"Parameters\": {\n \"ProjectName\": \"terra-ci-runner\",\n \"EnvironmentVariablesOverride\": [\n {\n \"Name\": \"TERRA_CI_BUILD_NAME\",\n \"Value.$\": \"$$.Execution.Name\"\n },\n {\n \"Name\": \"TERRA_CI_RESOURCE\",\n \"Value.$\": \"$.build.environment.terra_ci_resource\"\n }\n ]\n },\n \"End\": true\n },\n \"PlanBranch\": {\n \"Type\": \"Task\",\n \"Resource\": \"arn:aws:states:::codebuild:startBuild.sync\",\n \"Parameters\": {\n \"ProjectName\": \"terra-ci-runner\",\n \"SourceVersion.$\": \"$.build.sourceversion\",\n \"EnvironmentVariablesOverride\": [\n {\n \"Name\": \"TERRA_CI_RESOURCE\",\n \"Value.$\": \"$.build.environment.terra_ci_resource\"\n }\n ]\n },\n \"End\": true\n }\n }\n}\n", + "name": "terra-ci-runner", + "tags": null, + "type": "STANDARD" + }, + "after_unknown": { + "arn": true, + "creation_date": true, + "id": true, + "logging_configuration": true, + "role_arn": true, + "status": true + }, + "before_sensitive": false, + "after_sensitive": { + "logging_configuration": [] + } + } + }, + { + "address": "aws_route.private[\"1\"]", + "mode": "managed", + "type": "aws_route", + "name": "private", + "index": 1, + "provider_name": "registry.terraform.io/hashicorp/aws", + "change": { + "actions": [ + "create" + ], + "before": null, + "after": { + "carrier_gateway_id": null, + "destination_cidr_block": "172.25.16.0/20", + "destination_ipv6_cidr_block": null, + "destination_prefix_list_id": null, + "egress_only_gateway_id": null, + "gateway_id": null, + "local_gateway_id": null, + "nat_gateway_id": null, + "route_table_id": "rtb-00cf8381520103cfb", + "timeouts": null, + "transit_gateway_id": "tgw-0f68a4f2c58772c51", + "vpc_endpoint_id": null, + "vpc_peering_connection_id": null + }, + "after_unknown": { + "id": true, + "instance_id": true, + "instance_owner_id": true, + "network_interface_id": true, + "origin": true, + "state": true + }, + "before_sensitive": false, + "after_sensitive": {} + } + }, + { + "address": "aws_route.private[\"rtb-00cf8381520103cfb\"]", + "mode": "managed", + "type": "aws_route", + "name": "private", + "index": "rtb-00cf8381520103cfb", + "provider_name": "registry.terraform.io/hashicorp/aws", + "change": { + "actions": [ + "create" + ], + "before": null, + "after": { + "carrier_gateway_id": null, + "destination_cidr_block": "172.25.16.0/20", + "destination_ipv6_cidr_block": null, + "destination_prefix_list_id": null, + "egress_only_gateway_id": null, + "gateway_id": null, + "local_gateway_id": null, + "nat_gateway_id": null, + "route_table_id": "rtb-00cf8381520103cfb", + "timeouts": null, + "transit_gateway_id": "tgw-0f68a4f2c58772c51", + "vpc_endpoint_id": null, + "vpc_peering_connection_id": null + }, + "after_unknown": { + "id": true, + "instance_id": true, + "instance_owner_id": true, + "network_interface_id": true, + "origin": true, + "state": true + }, + "before_sensitive": false, + "after_sensitive": {} + } + } + ], + "prior_state": { + "format_version": "0.1", + "terraform_version": "0.15.0", + "values": { + "root_module": { + "resources": [ + { + "address": "data.aws_caller_identity.current", + "mode": "data", + "type": "aws_caller_identity", + "name": "current", + "provider_name": "registry.terraform.io/hashicorp/aws", + "schema_version": 0, + "values": { + "account_id": "719261439472", + "arn": "arn:aws:sts::719261439472:assumed-role/ci/1620222847597477484", + "id": "719261439472", + "user_id": "AROA2O52SSXYLVFYURBIV:1620222847597477484" + } + }, + { + "address": "data.template_file.terra_ci", + "mode": "data", + "type": "template_file", + "name": "terra_ci", + "provider_name": "registry.terraform.io/hashicorp/template", + "schema_version": 0, + "values": { + "filename": null, + "id": "64e36ed71e7270140dde96fec9c89d1d55ae5a6e91f7c0be15170200dcf9481b", + "rendered": "version: 0.2\nphases:\n install:\n commands:\n - make install_tools\n build:\n commands:\n - make plan_local resource=$TERRA_CI_RESOURCE\nartifacts:\n files:\n - ./tfplan\n name: $TERRA_CI_BUILD_NAME\n\n", + "template": "version: 0.2\nphases:\n install:\n commands:\n - make install_tools\n build:\n commands:\n - make plan_local resource=$TERRA_CI_RESOURCE\nartifacts:\n files:\n - ./tfplan\n name: $TERRA_CI_BUILD_NAME\n\n", + "vars": null + } + } + ] + } + } + }, + "configuration": { + "provider_config": { + "aws": { + "name": "aws", + "expressions": { + "allowed_account_ids": { + "constant_value": [ + "719261439472" + ] + }, + "assume_role": [ + { + "role_arn": { + "constant_value": "arn:aws:iam::719261439472:role/ci" + } + } + ], + "region": { + "constant_value": "eu-west-1" + } + } + } + }, + "root_module": { + "resources": [ + { + "address": "aws_codebuild_project.terra_ci", + "mode": "managed", + "type": "aws_codebuild_project", + "name": "terra_ci", + "provider_config_key": "aws", + "expressions": { + "artifacts": [ + { + "location": { + "references": [ + "aws_s3_bucket.terra_ci" + ] + }, + "type": { + "constant_value": "S3" + } + } + ], + "build_timeout": { + "constant_value": "10" + }, + "description": { + "constant_value": "Deploy environment configuration" + }, + "environment": [ + { + "compute_type": { + "constant_value": "BUILD_GENERAL1_SMALL" + }, + "image": { + "constant_value": "aws/codebuild/amazonlinux2-x86_64-standard:2.0" + }, + "image_pull_credentials_type": { + "constant_value": "CODEBUILD" + }, + "privileged_mode": { + "constant_value": false + }, + "type": { + "constant_value": "LINUX_CONTAINER" + } + } + ], + "logs_config": [ + { + "cloudwatch_logs": [ + { + "status": { + "constant_value": "ENABLED" + } + } + ], + "s3_logs": [ + { + "encryption_disabled": { + "constant_value": false + }, + "status": { + "constant_value": "DISABLED" + } + } + ] + } + ], + "name": { + "constant_value": "terra-ci-runner" + }, + "service_role": { + "references": [ + "aws_iam_role.terra_ci_job" + ] + }, + "source": [ + { + "buildspec": { + "references": [ + "data.template_file.terra_ci" + ] + }, + "git_clone_depth": { + "constant_value": 1 + }, + "insecure_ssl": { + "constant_value": false + }, + "location": { + "references": [ + "var.repo_url" + ] + }, + "report_build_status": { + "constant_value": false + }, + "type": { + "constant_value": "GITHUB" + } + } + ] + }, + "schema_version": 0 + }, + { + "address": "aws_iam_role.terra_ci_job", + "mode": "managed", + "type": "aws_iam_role", + "name": "terra_ci_job", + "provider_config_key": "aws", + "expressions": { + "assume_role_policy": { + "constant_value": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Principal\": {\n \"Service\": \"codebuild.amazonaws.com\"\n },\n \"Action\": \"sts:AssumeRole\"\n }\n ]\n}\n" + }, + "name": { + "constant_value": "terra_ci_job" + } + }, + "schema_version": 0 + }, + { + "address": "aws_iam_role.terra_ci_runner", + "mode": "managed", + "type": "aws_iam_role", + "name": "terra_ci_runner", + "provider_config_key": "aws", + "expressions": { + "assume_role_policy": { + "constant_value": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Principal\": {\n \"Service\": \"states.amazonaws.com\"\n },\n \"Action\": \"sts:AssumeRole\"\n }\n ]\n}\n" + }, + "name": { + "constant_value": "terra_ci_runner" + } + }, + "schema_version": 0 + }, + { + "address": "aws_iam_role_policy.terra_ci_job", + "mode": "managed", + "type": "aws_iam_role_policy", + "name": "terra_ci_job", + "provider_config_key": "aws", + "expressions": { + "policy": { + "references": [ + "data.aws_caller_identity.current", + "aws_s3_bucket.terra_ci", + "aws_s3_bucket.terra_ci" + ] + }, + "role": { + "references": [ + "aws_iam_role.terra_ci_job" + ] + } + }, + "schema_version": 0 + }, + { + "address": "aws_iam_role_policy.terra_ci_runner", + "mode": "managed", + "type": "aws_iam_role_policy", + "name": "terra_ci_runner", + "provider_config_key": "aws", + "expressions": { + "policy": { + "references": [ + "aws_codebuild_project.terra_ci", + "var.aws_region", + "data.aws_caller_identity.current" + ] + }, + "role": { + "references": [ + "aws_iam_role.terra_ci_runner" + ] + } + }, + "schema_version": 0 + }, + { + "address": "aws_iam_role_policy_attachment.terra_ci_job_ecr_access", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "terra_ci_job_ecr_access", + "provider_config_key": "aws", + "expressions": { + "policy_arn": { + "constant_value": "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryPowerUser" + }, + "role": { + "references": [ + "aws_iam_role.terra_ci_job" + ] + } + }, + "schema_version": 0 + }, + { + "address": "aws_s3_bucket.terra_ci", + "mode": "managed", + "type": "aws_s3_bucket", + "name": "terra_ci", + "provider_config_key": "aws", + "expressions": { + "acl": { + "constant_value": "private" + }, + "bucket": { + "references": [ + "var.aws_region", + "var.serial_number" + ] + }, + "server_side_encryption_configuration": [ + { + "rule": [ + { + "apply_server_side_encryption_by_default": [ + { + "sse_algorithm": { + "constant_value": "aws:kms" + } + } + ], + "bucket_key_enabled": { + "constant_value": false + } + } + ] + } + ] + }, + "schema_version": 0 + }, + { + "address": "aws_sfn_state_machine.terra_ci_runner", + "mode": "managed", + "type": "aws_sfn_state_machine", + "name": "terra_ci_runner", + "provider_config_key": "aws", + "expressions": { + "definition": { + "references": [ + "aws_codebuild_project.terra_ci", + "aws_codebuild_project.terra_ci" + ] + }, + "name": { + "constant_value": "terra-ci-runner" + }, + "role_arn": { + "references": [ + "aws_iam_role.terra_ci_runner" + ] + } + }, + "schema_version": 0 + }, + { + "address": "data.aws_caller_identity.current", + "mode": "data", + "type": "aws_caller_identity", + "name": "current", + "provider_config_key": "aws", + "schema_version": 0 + }, + { + "address": "data.template_file.terra_ci", + "mode": "data", + "type": "template_file", + "name": "terra_ci", + "provider_config_key": "template", + "expressions": { + "template": {} + }, + "schema_version": 0 + } + ], + "variables": { + "aws_region": {}, + "repo_url": {}, + "serial_number": {} + } + } + } + } diff --git a/tests/test_files/tf_content_generator_files/tfplan-destroy-example/tf_content.txt b/tests/test_files/tf_content_generator_files/tfplan-destroy-example/tf_content.txt new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_files/tf_content_generator_files/tfplan-destroy-example/tfplan.json b/tests/test_files/tf_content_generator_files/tfplan-destroy-example/tfplan.json new file mode 100644 index 00000000..88176ec9 --- /dev/null +++ b/tests/test_files/tf_content_generator_files/tfplan-destroy-example/tfplan.json @@ -0,0 +1,1082 @@ +{ + "comment-for-reader": "THIS TERRAFORM FILE REPRESENTS A REAL TF-PLAN OUTPUT FOR DELETED INFRASTRUCTRE", + "format_version": "0.1", + "terraform_version": "0.15.0", + "variables": { + "aws_region": { + "value": "eu-west-1" + }, + "repo_url": { + "value": "https://github.com/p0tr3c-terraform/terra-ci-single-account.git" + }, + "serial_number": { + "value": "000002" + } + }, + "planned_values": { + "root_module": {} + }, + "resource_changes": [ + { + "address": "aws_codebuild_project.terra_ci", + "mode": "managed", + "type": "aws_codebuild_project", + "name": "terra_ci", + "provider_name": "registry.terraform.io/hashicorp/aws", + "change": { + "actions": [ + "delete" + ], + "before": { + "arn": "arn:aws:codebuild:eu-west-1:719261439472:project/terra-ci-runner", + "artifacts": [ + { + "artifact_identifier": "", + "encryption_disabled": false, + "location": "terra-ci-artifacts-eu-west-1-000002", + "name": "terra-ci-runner", + "namespace_type": "NONE", + "override_artifact_name": false, + "packaging": "NONE", + "path": "", + "type": "S3" + } + ], + "badge_enabled": false, + "badge_url": "", + "build_timeout": 10, + "cache": [ + { + "location": "", + "modes": [], + "type": "NO_CACHE" + } + ], + "description": "Deploy environment configuration", + "encryption_key": "arn:aws:kms:eu-west-1:719261439472:alias/aws/s3", + "environment": [ + { + "certificate": "", + "compute_type": "BUILD_GENERAL1_SMALL", + "environment_variable": [], + "image": "aws/codebuild/amazonlinux2-x86_64-standard:2.0", + "image_pull_credentials_type": "CODEBUILD", + "privileged_mode": false, + "registry_credential": [], + "type": "LINUX_CONTAINER" + } + ], + "id": "arn:aws:codebuild:eu-west-1:719261439472:project/terra-ci-runner", + "logs_config": [ + { + "cloudwatch_logs": [ + { + "group_name": "", + "status": "ENABLED", + "stream_name": "" + } + ], + "s3_logs": [ + { + "encryption_disabled": false, + "location": "", + "status": "DISABLED" + } + ] + } + ], + "name": "terra-ci-runner", + "queued_timeout": 480, + "secondary_artifacts": [], + "secondary_sources": [], + "service_role": "arn:aws:iam::719261439472:role/terra_ci_job", + "source": [ + { + "auth": [], + "buildspec": "version: 0.2\nphases:\n install:\n commands:\n - make install_tools\n build:\n commands:\n - make plan_local resource=$TERRA_CI_RESOURCE\nartifacts:\n files:\n - ./tfplan\n name: $TERRA_CI_BUILD_NAME\n\n", + "git_clone_depth": 1, + "git_submodules_config": [], + "insecure_ssl": false, + "location": "https://github.com/p0tr3c-terraform/terra-ci-single-account.git", + "report_build_status": false, + "type": "GITHUB" + } + ], + "source_version": "", + "tags": {}, + "vpc_config": [] + }, + "after": null, + "after_unknown": {}, + "before_sensitive": { + "artifacts": [ + {} + ], + "cache": [ + { + "modes": [] + } + ], + "environment": [ + { + "environment_variable": [], + "registry_credential": [] + } + ], + "logs_config": [ + { + "cloudwatch_logs": [ + {} + ], + "s3_logs": [ + {} + ] + } + ], + "secondary_artifacts": [], + "secondary_sources": [], + "source": [ + { + "auth": [], + "git_submodules_config": [] + } + ], + "tags": {}, + "vpc_config": [] + }, + "after_sensitive": false + } + }, + { + "address": "aws_iam_role.terra_ci_job", + "mode": "managed", + "type": "aws_iam_role", + "name": "terra_ci_job", + "provider_name": "registry.terraform.io/hashicorp/aws", + "change": { + "actions": [ + "delete" + ], + "before": { + "arn": "arn:aws:iam::719261439472:role/terra_ci_job", + "assume_role_policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"codebuild.amazonaws.com\"},\"Action\":\"sts:AssumeRole\"}]}", + "create_date": "2021-05-05T13:59:07Z", + "description": "", + "force_detach_policies": false, + "id": "terra_ci_job", + "inline_policy": [ + { + "name": "terraform-20210505135914255500000001", + "policy": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Resource\": \"arn:aws:iam::719261439472:role/ci\"\n },\n {\n \"Effect\": \"Allow\",\n \"Resource\": [\n \"*\"\n ],\n \"Action\": [\n \"logs:CreateLogGroup\",\n \"logs:CreateLogStream\",\n \"logs:PutLogEvents\"\n ]\n },\n {\n \"Effect\": \"Allow\",\n \"Resource\": [\n \"arn:aws:s3:::terra-ci-artifacts-eu-west-1-000002\",\n \"arn:aws:s3:::terra-ci-artifacts-eu-west-1-000002/*\"\n ],\n \"Action\": [\n \"s3:ListBucket\",\n \"s3:*Object\"\n ]\n }\n ]\n}\n" + } + ], + "managed_policy_arns": [ + "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryPowerUser" + ], + "max_session_duration": 3600, + "name": "terra_ci_job", + "name_prefix": null, + "path": "/", + "permissions_boundary": null, + "tags": {}, + "unique_id": "AROA2O52SSXYMEA4VGINT" + }, + "after": null, + "after_unknown": {}, + "before_sensitive": { + "inline_policy": [ + {} + ], + "managed_policy_arns": [ + false + ], + "tags": {} + }, + "after_sensitive": false + } + }, + { + "address": "aws_iam_role.terra_ci_runner", + "mode": "managed", + "type": "aws_iam_role", + "name": "terra_ci_runner", + "provider_name": "registry.terraform.io/hashicorp/aws", + "change": { + "actions": [ + "delete" + ], + "before": { + "arn": "arn:aws:iam::719261439472:role/terra_ci_runner", + "assume_role_policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"states.amazonaws.com\"},\"Action\":\"sts:AssumeRole\"}]}", + "create_date": "2021-05-05T13:59:07Z", + "description": "", + "force_detach_policies": false, + "id": "terra_ci_runner", + "inline_policy": [ + { + "name": "terraform-20210505135918902900000003", + "policy": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"codebuild:StartBuild\",\n \"codebuild:StopBuild\",\n \"codebuild:BatchGetBuilds\"\n ],\n \"Resource\": [\n \"arn:aws:codebuild:eu-west-1:719261439472:project/terra-ci-runner\"\n ]\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"events:PutTargets\",\n \"events:PutRule\",\n \"events:DescribeRule\"\n ],\n \"Resource\": [\n \"arn:aws:events:eu-west-1:719261439472:rule/StepFunctionsGetEventForCodeBuildStartBuildRule\"\n ]\n }\n ]\n}\n" + } + ], + "managed_policy_arns": [], + "max_session_duration": 3600, + "name": "terra_ci_runner", + "name_prefix": null, + "path": "/", + "permissions_boundary": null, + "tags": {}, + "unique_id": "AROA2O52SSXYPQLY6HIHV" + }, + "after": null, + "after_unknown": {}, + "before_sensitive": { + "inline_policy": [ + {} + ], + "managed_policy_arns": [], + "tags": {} + }, + "after_sensitive": false + } + }, + { + "address": "aws_iam_role_policy.terra_ci_job", + "mode": "managed", + "type": "aws_iam_role_policy", + "name": "terra_ci_job", + "provider_name": "registry.terraform.io/hashicorp/aws", + "change": { + "actions": [ + "delete" + ], + "before": { + "id": "terra_ci_job:terraform-20210505135914255500000001", + "name": "terraform-20210505135914255500000001", + "name_prefix": null, + "policy": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Resource\": \"arn:aws:iam::719261439472:role/ci\"\n },\n {\n \"Effect\": \"Allow\",\n \"Resource\": [\n \"*\"\n ],\n \"Action\": [\n \"logs:CreateLogGroup\",\n \"logs:CreateLogStream\",\n \"logs:PutLogEvents\"\n ]\n },\n {\n \"Effect\": \"Allow\",\n \"Resource\": [\n \"arn:aws:s3:::terra-ci-artifacts-eu-west-1-000002\",\n \"arn:aws:s3:::terra-ci-artifacts-eu-west-1-000002/*\"\n ],\n \"Action\": [\n \"s3:ListBucket\",\n \"s3:*Object\"\n ]\n }\n ]\n}\n", + "role": "terra_ci_job" + }, + "after": null, + "after_unknown": {}, + "before_sensitive": {}, + "after_sensitive": false + } + }, + { + "address": "aws_iam_role_policy.terra_ci_runner", + "mode": "managed", + "type": "aws_iam_role_policy", + "name": "terra_ci_runner", + "provider_name": "registry.terraform.io/hashicorp/aws", + "change": { + "actions": [ + "delete" + ], + "before": { + "id": "terra_ci_runner:terraform-20210505135918902900000003", + "name": "terraform-20210505135918902900000003", + "name_prefix": null, + "policy": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"codebuild:StartBuild\",\n \"codebuild:StopBuild\",\n \"codebuild:BatchGetBuilds\"\n ],\n \"Resource\": [\n \"arn:aws:codebuild:eu-west-1:719261439472:project/terra-ci-runner\"\n ]\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"events:PutTargets\",\n \"events:PutRule\",\n \"events:DescribeRule\"\n ],\n \"Resource\": [\n \"arn:aws:events:eu-west-1:719261439472:rule/StepFunctionsGetEventForCodeBuildStartBuildRule\"\n ]\n }\n ]\n}\n", + "role": "terra_ci_runner" + }, + "after": null, + "after_unknown": {}, + "before_sensitive": {}, + "after_sensitive": false + } + }, + { + "address": "aws_iam_role_policy_attachment.terra_ci_job_ecr_access", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "terra_ci_job_ecr_access", + "provider_name": "registry.terraform.io/hashicorp/aws", + "change": { + "actions": [ + "delete" + ], + "before": { + "id": "terra_ci_job-20210505135914799900000002", + "policy_arn": "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryPowerUser", + "role": "terra_ci_job" + }, + "after": null, + "after_unknown": {}, + "before_sensitive": {}, + "after_sensitive": false + } + }, + { + "address": "aws_s3_bucket.terra_ci", + "mode": "managed", + "type": "aws_s3_bucket", + "name": "terra_ci", + "provider_name": "registry.terraform.io/hashicorp/aws", + "change": { + "actions": [ + "delete" + ], + "before": { + "acceleration_status": "", + "acl": "private", + "arn": "arn:aws:s3:::terra-ci-artifacts-eu-west-1-000002", + "bucket": "terra-ci-artifacts-eu-west-1-000002", + "bucket_domain_name": "terra-ci-artifacts-eu-west-1-000002.s3.amazonaws.com", + "bucket_prefix": null, + "bucket_regional_domain_name": "terra-ci-artifacts-eu-west-1-000002.s3.eu-west-1.amazonaws.com", + "cors_rule": [], + "force_destroy": false, + "grant": [], + "hosted_zone_id": "Z1BKCTXD74EZPE", + "id": "terra-ci-artifacts-eu-west-1-000002", + "lifecycle_rule": [], + "logging": [], + "object_lock_configuration": [], + "policy": null, + "region": "eu-west-1", + "replication_configuration": [], + "request_payer": "BucketOwner", + "server_side_encryption_configuration": [ + { + "rule": [ + { + "apply_server_side_encryption_by_default": [ + { + "kms_master_key_id": "", + "sse_algorithm": "aws:kms" + } + ], + "bucket_key_enabled": false + } + ] + } + ], + "tags": {}, + "versioning": [ + { + "enabled": false, + "mfa_delete": false + } + ], + "website": [], + "website_domain": null, + "website_endpoint": null + }, + "after": null, + "after_unknown": {}, + "before_sensitive": { + "cors_rule": [], + "grant": [], + "lifecycle_rule": [], + "logging": [], + "object_lock_configuration": [], + "replication_configuration": [], + "server_side_encryption_configuration": [ + { + "rule": [ + { + "apply_server_side_encryption_by_default": [ + {} + ] + } + ] + } + ], + "tags": {}, + "versioning": [ + {} + ], + "website": [] + }, + "after_sensitive": false + } + }, + { + "address": "aws_sfn_state_machine.terra_ci_runner", + "mode": "managed", + "type": "aws_sfn_state_machine", + "name": "terra_ci_runner", + "provider_name": "registry.terraform.io/hashicorp/aws", + "change": { + "actions": [ + "delete" + ], + "before": { + "arn": "arn:aws:states:eu-west-1:719261439472:stateMachine:terra-ci-runner", + "creation_date": "2021-05-05T13:59:28Z", + "definition": "{\n \"Comment\": \"Run Terragrunt Jobs\",\n \"StartAt\": \"OnBranch?\",\n \"States\": {\n \"OnBranch?\": {\n \"Type\": \"Choice\",\n \"Choices\": [\n {\n \"Variable\": \"$.build.sourceversion\",\n \"IsPresent\": true,\n \"Next\": \"PlanBranch\"\n }\n ],\n \"Default\": \"Plan\"\n },\n \"Plan\": {\n \"Type\": \"Task\",\n \"Resource\": \"arn:aws:states:::codebuild:startBuild.sync\",\n \"Parameters\": {\n \"ProjectName\": \"terra-ci-runner\",\n \"EnvironmentVariablesOverride\": [\n {\n \"Name\": \"TERRA_CI_BUILD_NAME\",\n \"Value.$\": \"$$.Execution.Name\"\n },\n {\n \"Name\": \"TERRA_CI_RESOURCE\",\n \"Value.$\": \"$.build.environment.terra_ci_resource\"\n }\n ]\n },\n \"End\": true\n },\n \"PlanBranch\": {\n \"Type\": \"Task\",\n \"Resource\": \"arn:aws:states:::codebuild:startBuild.sync\",\n \"Parameters\": {\n \"ProjectName\": \"terra-ci-runner\",\n \"SourceVersion.$\": \"$.build.sourceversion\",\n \"EnvironmentVariablesOverride\": [\n {\n \"Name\": \"TERRA_CI_RESOURCE\",\n \"Value.$\": \"$.build.environment.terra_ci_resource\"\n }\n ]\n },\n \"End\": true\n }\n }\n}\n", + "id": "arn:aws:states:eu-west-1:719261439472:stateMachine:terra-ci-runner", + "logging_configuration": [ + { + "include_execution_data": false, + "level": "OFF", + "log_destination": "" + } + ], + "name": "terra-ci-runner", + "role_arn": "arn:aws:iam::719261439472:role/terra_ci_runner", + "status": "ACTIVE", + "tags": {}, + "type": "STANDARD" + }, + "after": null, + "after_unknown": {}, + "before_sensitive": { + "logging_configuration": [ + {} + ], + "tags": {} + }, + "after_sensitive": false + } + } + ], + "prior_state": { + "format_version": "0.1", + "terraform_version": "0.15.0", + "values": { + "root_module": { + "resources": [ + { + "address": "aws_codebuild_project.terra_ci", + "mode": "managed", + "type": "aws_codebuild_project", + "name": "terra_ci", + "provider_name": "registry.terraform.io/hashicorp/aws", + "schema_version": 0, + "values": { + "arn": "arn:aws:codebuild:eu-west-1:719261439472:project/terra-ci-runner", + "artifacts": [ + { + "artifact_identifier": "", + "encryption_disabled": false, + "location": "terra-ci-artifacts-eu-west-1-000002", + "name": "terra-ci-runner", + "namespace_type": "NONE", + "override_artifact_name": false, + "packaging": "NONE", + "path": "", + "type": "S3" + } + ], + "badge_enabled": false, + "badge_url": "", + "build_timeout": 10, + "cache": [ + { + "location": "", + "modes": [], + "type": "NO_CACHE" + } + ], + "description": "Deploy environment configuration", + "encryption_key": "arn:aws:kms:eu-west-1:719261439472:alias/aws/s3", + "environment": [ + { + "certificate": "", + "compute_type": "BUILD_GENERAL1_SMALL", + "environment_variable": [], + "image": "aws/codebuild/amazonlinux2-x86_64-standard:2.0", + "image_pull_credentials_type": "CODEBUILD", + "privileged_mode": false, + "registry_credential": [], + "type": "LINUX_CONTAINER" + } + ], + "id": "arn:aws:codebuild:eu-west-1:719261439472:project/terra-ci-runner", + "logs_config": [ + { + "cloudwatch_logs": [ + { + "group_name": "", + "status": "ENABLED", + "stream_name": "" + } + ], + "s3_logs": [ + { + "encryption_disabled": false, + "location": "", + "status": "DISABLED" + } + ] + } + ], + "name": "terra-ci-runner", + "queued_timeout": 480, + "secondary_artifacts": [], + "secondary_sources": [], + "service_role": "arn:aws:iam::719261439472:role/terra_ci_job", + "source": [ + { + "auth": [], + "buildspec": "version: 0.2\nphases:\n install:\n commands:\n - make install_tools\n build:\n commands:\n - make plan_local resource=$TERRA_CI_RESOURCE\nartifacts:\n files:\n - ./tfplan\n name: $TERRA_CI_BUILD_NAME\n\n", + "git_clone_depth": 1, + "git_submodules_config": [], + "insecure_ssl": false, + "location": "https://github.com/p0tr3c-terraform/terra-ci-single-account.git", + "report_build_status": false, + "type": "GITHUB" + } + ], + "source_version": "", + "tags": {}, + "vpc_config": [] + }, + "depends_on": [ + "aws_iam_role.terra_ci_job", + "aws_s3_bucket.terra_ci", + "data.template_file.terra_ci" + ] + }, + { + "address": "aws_iam_role.terra_ci_job", + "mode": "managed", + "type": "aws_iam_role", + "name": "terra_ci_job", + "provider_name": "registry.terraform.io/hashicorp/aws", + "schema_version": 0, + "values": { + "arn": "arn:aws:iam::719261439472:role/terra_ci_job", + "assume_role_policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"codebuild.amazonaws.com\"},\"Action\":\"sts:AssumeRole\"}]}", + "create_date": "2021-05-05T13:59:07Z", + "description": "", + "force_detach_policies": false, + "id": "terra_ci_job", + "inline_policy": [ + { + "name": "terraform-20210505135914255500000001", + "policy": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Resource\": \"arn:aws:iam::719261439472:role/ci\"\n },\n {\n \"Effect\": \"Allow\",\n \"Resource\": [\n \"*\"\n ],\n \"Action\": [\n \"logs:CreateLogGroup\",\n \"logs:CreateLogStream\",\n \"logs:PutLogEvents\"\n ]\n },\n {\n \"Effect\": \"Allow\",\n \"Resource\": [\n \"arn:aws:s3:::terra-ci-artifacts-eu-west-1-000002\",\n \"arn:aws:s3:::terra-ci-artifacts-eu-west-1-000002/*\"\n ],\n \"Action\": [\n \"s3:ListBucket\",\n \"s3:*Object\"\n ]\n }\n ]\n}\n" + } + ], + "managed_policy_arns": [ + "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryPowerUser" + ], + "max_session_duration": 3600, + "name": "terra_ci_job", + "name_prefix": null, + "path": "/", + "permissions_boundary": null, + "tags": {}, + "unique_id": "AROA2O52SSXYMEA4VGINT" + } + }, + { + "address": "aws_iam_role.terra_ci_runner", + "mode": "managed", + "type": "aws_iam_role", + "name": "terra_ci_runner", + "provider_name": "registry.terraform.io/hashicorp/aws", + "schema_version": 0, + "values": { + "arn": "arn:aws:iam::719261439472:role/terra_ci_runner", + "assume_role_policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"states.amazonaws.com\"},\"Action\":\"sts:AssumeRole\"}]}", + "create_date": "2021-05-05T13:59:07Z", + "description": "", + "force_detach_policies": false, + "id": "terra_ci_runner", + "inline_policy": [ + { + "name": "terraform-20210505135918902900000003", + "policy": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"codebuild:StartBuild\",\n \"codebuild:StopBuild\",\n \"codebuild:BatchGetBuilds\"\n ],\n \"Resource\": [\n \"arn:aws:codebuild:eu-west-1:719261439472:project/terra-ci-runner\"\n ]\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"events:PutTargets\",\n \"events:PutRule\",\n \"events:DescribeRule\"\n ],\n \"Resource\": [\n \"arn:aws:events:eu-west-1:719261439472:rule/StepFunctionsGetEventForCodeBuildStartBuildRule\"\n ]\n }\n ]\n}\n" + } + ], + "managed_policy_arns": [], + "max_session_duration": 3600, + "name": "terra_ci_runner", + "name_prefix": null, + "path": "/", + "permissions_boundary": null, + "tags": {}, + "unique_id": "AROA2O52SSXYPQLY6HIHV" + } + }, + { + "address": "aws_iam_role_policy.terra_ci_job", + "mode": "managed", + "type": "aws_iam_role_policy", + "name": "terra_ci_job", + "provider_name": "registry.terraform.io/hashicorp/aws", + "schema_version": 0, + "values": { + "id": "terra_ci_job:terraform-20210505135914255500000001", + "name": "terraform-20210505135914255500000001", + "name_prefix": null, + "policy": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Resource\": \"arn:aws:iam::719261439472:role/ci\"\n },\n {\n \"Effect\": \"Allow\",\n \"Resource\": [\n \"*\"\n ],\n \"Action\": [\n \"logs:CreateLogGroup\",\n \"logs:CreateLogStream\",\n \"logs:PutLogEvents\"\n ]\n },\n {\n \"Effect\": \"Allow\",\n \"Resource\": [\n \"arn:aws:s3:::terra-ci-artifacts-eu-west-1-000002\",\n \"arn:aws:s3:::terra-ci-artifacts-eu-west-1-000002/*\"\n ],\n \"Action\": [\n \"s3:ListBucket\",\n \"s3:*Object\"\n ]\n }\n ]\n}\n", + "role": "terra_ci_job" + }, + "depends_on": [ + "aws_iam_role.terra_ci_job", + "aws_s3_bucket.terra_ci", + "data.aws_caller_identity.current" + ] + }, + { + "address": "aws_iam_role_policy.terra_ci_runner", + "mode": "managed", + "type": "aws_iam_role_policy", + "name": "terra_ci_runner", + "provider_name": "registry.terraform.io/hashicorp/aws", + "schema_version": 0, + "values": { + "id": "terra_ci_runner:terraform-20210505135918902900000003", + "name": "terraform-20210505135918902900000003", + "name_prefix": null, + "policy": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"codebuild:StartBuild\",\n \"codebuild:StopBuild\",\n \"codebuild:BatchGetBuilds\"\n ],\n \"Resource\": [\n \"arn:aws:codebuild:eu-west-1:719261439472:project/terra-ci-runner\"\n ]\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"events:PutTargets\",\n \"events:PutRule\",\n \"events:DescribeRule\"\n ],\n \"Resource\": [\n \"arn:aws:events:eu-west-1:719261439472:rule/StepFunctionsGetEventForCodeBuildStartBuildRule\"\n ]\n }\n ]\n}\n", + "role": "terra_ci_runner" + }, + "depends_on": [ + "aws_codebuild_project.terra_ci", + "aws_iam_role.terra_ci_job", + "aws_iam_role.terra_ci_runner", + "aws_s3_bucket.terra_ci", + "data.aws_caller_identity.current", + "data.template_file.terra_ci" + ] + }, + { + "address": "aws_iam_role_policy_attachment.terra_ci_job_ecr_access", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "terra_ci_job_ecr_access", + "provider_name": "registry.terraform.io/hashicorp/aws", + "schema_version": 0, + "values": { + "id": "terra_ci_job-20210505135914799900000002", + "policy_arn": "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryPowerUser", + "role": "terra_ci_job" + }, + "depends_on": [ + "aws_iam_role.terra_ci_job" + ] + }, + { + "address": "aws_s3_bucket.terra_ci", + "mode": "managed", + "type": "aws_s3_bucket", + "name": "terra_ci", + "provider_name": "registry.terraform.io/hashicorp/aws", + "schema_version": 0, + "values": { + "acceleration_status": "", + "acl": "private", + "arn": "arn:aws:s3:::terra-ci-artifacts-eu-west-1-000002", + "bucket": "terra-ci-artifacts-eu-west-1-000002", + "bucket_domain_name": "terra-ci-artifacts-eu-west-1-000002.s3.amazonaws.com", + "bucket_prefix": null, + "bucket_regional_domain_name": "terra-ci-artifacts-eu-west-1-000002.s3.eu-west-1.amazonaws.com", + "cors_rule": [], + "force_destroy": false, + "grant": [], + "hosted_zone_id": "Z1BKCTXD74EZPE", + "id": "terra-ci-artifacts-eu-west-1-000002", + "lifecycle_rule": [], + "logging": [], + "object_lock_configuration": [], + "policy": null, + "region": "eu-west-1", + "replication_configuration": [], + "request_payer": "BucketOwner", + "server_side_encryption_configuration": [ + { + "rule": [ + { + "apply_server_side_encryption_by_default": [ + { + "kms_master_key_id": "", + "sse_algorithm": "aws:kms" + } + ], + "bucket_key_enabled": false + } + ] + } + ], + "tags": {}, + "versioning": [ + { + "enabled": false, + "mfa_delete": false + } + ], + "website": [], + "website_domain": null, + "website_endpoint": null + } + }, + { + "address": "aws_sfn_state_machine.terra_ci_runner", + "mode": "managed", + "type": "aws_sfn_state_machine", + "name": "terra_ci_runner", + "provider_name": "registry.terraform.io/hashicorp/aws", + "schema_version": 0, + "values": { + "arn": "arn:aws:states:eu-west-1:719261439472:stateMachine:terra-ci-runner", + "creation_date": "2021-05-05T13:59:28Z", + "definition": "{\n \"Comment\": \"Run Terragrunt Jobs\",\n \"StartAt\": \"OnBranch?\",\n \"States\": {\n \"OnBranch?\": {\n \"Type\": \"Choice\",\n \"Choices\": [\n {\n \"Variable\": \"$.build.sourceversion\",\n \"IsPresent\": true,\n \"Next\": \"PlanBranch\"\n }\n ],\n \"Default\": \"Plan\"\n },\n \"Plan\": {\n \"Type\": \"Task\",\n \"Resource\": \"arn:aws:states:::codebuild:startBuild.sync\",\n \"Parameters\": {\n \"ProjectName\": \"terra-ci-runner\",\n \"EnvironmentVariablesOverride\": [\n {\n \"Name\": \"TERRA_CI_BUILD_NAME\",\n \"Value.$\": \"$$.Execution.Name\"\n },\n {\n \"Name\": \"TERRA_CI_RESOURCE\",\n \"Value.$\": \"$.build.environment.terra_ci_resource\"\n }\n ]\n },\n \"End\": true\n },\n \"PlanBranch\": {\n \"Type\": \"Task\",\n \"Resource\": \"arn:aws:states:::codebuild:startBuild.sync\",\n \"Parameters\": {\n \"ProjectName\": \"terra-ci-runner\",\n \"SourceVersion.$\": \"$.build.sourceversion\",\n \"EnvironmentVariablesOverride\": [\n {\n \"Name\": \"TERRA_CI_RESOURCE\",\n \"Value.$\": \"$.build.environment.terra_ci_resource\"\n }\n ]\n },\n \"End\": true\n }\n }\n}\n", + "id": "arn:aws:states:eu-west-1:719261439472:stateMachine:terra-ci-runner", + "logging_configuration": [ + { + "include_execution_data": false, + "level": "OFF", + "log_destination": "" + } + ], + "name": "terra-ci-runner", + "role_arn": "arn:aws:iam::719261439472:role/terra_ci_runner", + "status": "ACTIVE", + "tags": {}, + "type": "STANDARD" + }, + "depends_on": [ + "aws_iam_role.terra_ci_job", + "aws_iam_role.terra_ci_runner", + "aws_s3_bucket.terra_ci", + "data.template_file.terra_ci", + "aws_codebuild_project.terra_ci" + ] + }, + { + "address": "data.aws_caller_identity.current", + "mode": "data", + "type": "aws_caller_identity", + "name": "current", + "provider_name": "registry.terraform.io/hashicorp/aws", + "schema_version": 0, + "values": { + "account_id": "719261439472", + "arn": "arn:aws:sts::719261439472:assumed-role/ci/1620223184813487213", + "id": "719261439472", + "user_id": "AROA2O52SSXYLVFYURBIV:1620223184813487213" + } + }, + { + "address": "data.template_file.terra_ci", + "mode": "data", + "type": "template_file", + "name": "terra_ci", + "provider_name": "registry.terraform.io/hashicorp/template", + "schema_version": 0, + "values": { + "filename": null, + "id": "64e36ed71e7270140dde96fec9c89d1d55ae5a6e91f7c0be15170200dcf9481b", + "rendered": "version: 0.2\nphases:\n install:\n commands:\n - make install_tools\n build:\n commands:\n - make plan_local resource=$TERRA_CI_RESOURCE\nartifacts:\n files:\n - ./tfplan\n name: $TERRA_CI_BUILD_NAME\n\n", + "template": "version: 0.2\nphases:\n install:\n commands:\n - make install_tools\n build:\n commands:\n - make plan_local resource=$TERRA_CI_RESOURCE\nartifacts:\n files:\n - ./tfplan\n name: $TERRA_CI_BUILD_NAME\n\n", + "vars": null + } + } + ] + } + } + }, + "configuration": { + "provider_config": { + "aws": { + "name": "aws", + "expressions": { + "allowed_account_ids": { + "constant_value": [ + "719261439472" + ] + }, + "assume_role": [ + { + "role_arn": { + "constant_value": "arn:aws:iam::719261439472:role/ci" + } + } + ], + "region": { + "constant_value": "eu-west-1" + } + } + } + }, + "root_module": { + "resources": [ + { + "address": "aws_codebuild_project.terra_ci", + "mode": "managed", + "type": "aws_codebuild_project", + "name": "terra_ci", + "provider_config_key": "aws", + "expressions": { + "artifacts": [ + { + "location": { + "references": [ + "aws_s3_bucket.terra_ci" + ] + }, + "type": { + "constant_value": "S3" + } + } + ], + "build_timeout": { + "constant_value": "10" + }, + "description": { + "constant_value": "Deploy environment configuration" + }, + "environment": [ + { + "compute_type": { + "constant_value": "BUILD_GENERAL1_SMALL" + }, + "image": { + "constant_value": "aws/codebuild/amazonlinux2-x86_64-standard:2.0" + }, + "image_pull_credentials_type": { + "constant_value": "CODEBUILD" + }, + "privileged_mode": { + "constant_value": false + }, + "type": { + "constant_value": "LINUX_CONTAINER" + } + } + ], + "logs_config": [ + { + "cloudwatch_logs": [ + { + "status": { + "constant_value": "ENABLED" + } + } + ], + "s3_logs": [ + { + "encryption_disabled": { + "constant_value": false + }, + "status": { + "constant_value": "DISABLED" + } + } + ] + } + ], + "name": { + "constant_value": "terra-ci-runner" + }, + "service_role": { + "references": [ + "aws_iam_role.terra_ci_job" + ] + }, + "source": [ + { + "buildspec": { + "references": [ + "data.template_file.terra_ci" + ] + }, + "git_clone_depth": { + "constant_value": 1 + }, + "insecure_ssl": { + "constant_value": false + }, + "location": { + "references": [ + "var.repo_url" + ] + }, + "report_build_status": { + "constant_value": false + }, + "type": { + "constant_value": "GITHUB" + } + } + ] + }, + "schema_version": 0 + }, + { + "address": "aws_iam_role.terra_ci_job", + "mode": "managed", + "type": "aws_iam_role", + "name": "terra_ci_job", + "provider_config_key": "aws", + "expressions": { + "assume_role_policy": { + "constant_value": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Principal\": {\n \"Service\": \"codebuild.amazonaws.com\"\n },\n \"Action\": \"sts:AssumeRole\"\n }\n ]\n}\n" + }, + "name": { + "constant_value": "terra_ci_job" + } + }, + "schema_version": 0 + }, + { + "address": "aws_iam_role.terra_ci_runner", + "mode": "managed", + "type": "aws_iam_role", + "name": "terra_ci_runner", + "provider_config_key": "aws", + "expressions": { + "assume_role_policy": { + "constant_value": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Principal\": {\n \"Service\": \"states.amazonaws.com\"\n },\n \"Action\": \"sts:AssumeRole\"\n }\n ]\n}\n" + }, + "name": { + "constant_value": "terra_ci_runner" + } + }, + "schema_version": 0 + }, + { + "address": "aws_iam_role_policy.terra_ci_job", + "mode": "managed", + "type": "aws_iam_role_policy", + "name": "terra_ci_job", + "provider_config_key": "aws", + "expressions": { + "policy": { + "references": [ + "data.aws_caller_identity.current", + "aws_s3_bucket.terra_ci", + "aws_s3_bucket.terra_ci" + ] + }, + "role": { + "references": [ + "aws_iam_role.terra_ci_job" + ] + } + }, + "schema_version": 0 + }, + { + "address": "aws_iam_role_policy.terra_ci_runner", + "mode": "managed", + "type": "aws_iam_role_policy", + "name": "terra_ci_runner", + "provider_config_key": "aws", + "expressions": { + "policy": { + "references": [ + "aws_codebuild_project.terra_ci", + "var.aws_region", + "data.aws_caller_identity.current" + ] + }, + "role": { + "references": [ + "aws_iam_role.terra_ci_runner" + ] + } + }, + "schema_version": 0 + }, + { + "address": "aws_iam_role_policy_attachment.terra_ci_job_ecr_access", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "terra_ci_job_ecr_access", + "provider_config_key": "aws", + "expressions": { + "policy_arn": { + "constant_value": "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryPowerUser" + }, + "role": { + "references": [ + "aws_iam_role.terra_ci_job" + ] + } + }, + "schema_version": 0 + }, + { + "address": "aws_s3_bucket.terra_ci", + "mode": "managed", + "type": "aws_s3_bucket", + "name": "terra_ci", + "provider_config_key": "aws", + "expressions": { + "acl": { + "constant_value": "private" + }, + "bucket": { + "references": [ + "var.aws_region", + "var.serial_number" + ] + }, + "server_side_encryption_configuration": [ + { + "rule": [ + { + "apply_server_side_encryption_by_default": [ + { + "sse_algorithm": { + "constant_value": "aws:kms" + } + } + ], + "bucket_key_enabled": { + "constant_value": false + } + } + ] + } + ] + }, + "schema_version": 0 + }, + { + "address": "aws_sfn_state_machine.terra_ci_runner", + "mode": "managed", + "type": "aws_sfn_state_machine", + "name": "terra_ci_runner", + "provider_config_key": "aws", + "expressions": { + "definition": { + "references": [ + "aws_codebuild_project.terra_ci", + "aws_codebuild_project.terra_ci" + ] + }, + "name": { + "constant_value": "terra-ci-runner" + }, + "role_arn": { + "references": [ + "aws_iam_role.terra_ci_runner" + ] + } + }, + "schema_version": 0 + }, + { + "address": "data.aws_caller_identity.current", + "mode": "data", + "type": "aws_caller_identity", + "name": "current", + "provider_config_key": "aws", + "schema_version": 0 + }, + { + "address": "data.template_file.terra_ci", + "mode": "data", + "type": "template_file", + "name": "terra_ci", + "provider_config_key": "template", + "expressions": { + "template": {} + }, + "schema_version": 0 + } + ], + "variables": { + "aws_region": {}, + "repo_url": {}, + "serial_number": {} + } + } + } + } diff --git a/tests/test_files/tf_content_generator_files/tfplan-false-var/tf_content.txt b/tests/test_files/tf_content_generator_files/tfplan-false-var/tf_content.txt new file mode 100644 index 00000000..5d902651 --- /dev/null +++ b/tests/test_files/tf_content_generator_files/tfplan-false-var/tf_content.txt @@ -0,0 +1,14 @@ +resource "aws_s3_bucket" "efrat-env-var-test" { + bucket = "efrat-env-var-test" + force_destroy = false + tags = null + timeouts = null +} + +resource "aws_s3_bucket_public_access_block" "efrat-env-var-test" { + block_public_acls = false + block_public_policy = true + ignore_public_acls = false + restrict_public_buckets = true +} + diff --git a/tests/test_files/tf_content_generator_files/tfplan-false-var/tfplan.json b/tests/test_files/tf_content_generator_files/tfplan-false-var/tfplan.json new file mode 100644 index 00000000..2c8d033c --- /dev/null +++ b/tests/test_files/tf_content_generator_files/tfplan-false-var/tfplan.json @@ -0,0 +1,218 @@ +{ + "format_version": "1.2", + "terraform_version": "1.5.4", + "variables": { + "IT_IS_FALSE": { + "value": "false" + } + }, + "planned_values": { + "root_module": { + "resources": [ + { + "address": "aws_s3_bucket.efrat-env-var-test", + "mode": "managed", + "type": "aws_s3_bucket", + "name": "efrat-env-var-test", + "provider_name": "registry.terraform.io/hashicorp/aws", + "schema_version": 0, + "values": { + "bucket": "efrat-env-var-test", + "force_destroy": false, + "tags": null, + "timeouts": null + }, + "sensitive_values": { + "cors_rule": [], + "grant": [], + "lifecycle_rule": [], + "logging": [], + "object_lock_configuration": [], + "replication_configuration": [], + "server_side_encryption_configuration": [], + "tags_all": {}, + "versioning": [], + "website": [] + } + }, + { + "address": "aws_s3_bucket_public_access_block.efrat-env-var-test", + "mode": "managed", + "type": "aws_s3_bucket_public_access_block", + "name": "efrat-env-var-test", + "provider_name": "registry.terraform.io/hashicorp/aws", + "schema_version": 0, + "values": { + "block_public_acls": false, + "block_public_policy": true, + "ignore_public_acls": false, + "restrict_public_buckets": true + }, + "sensitive_values": {} + } + ] + } + }, + "resource_changes": [ + { + "address": "aws_s3_bucket.efrat-env-var-test", + "mode": "managed", + "type": "aws_s3_bucket", + "name": "efrat-env-var-test", + "provider_name": "registry.terraform.io/hashicorp/aws", + "change": { + "actions": [ + "create" + ], + "before": null, + "after": { + "bucket": "efrat-env-var-test", + "force_destroy": false, + "tags": null, + "timeouts": null + }, + "after_unknown": { + "acceleration_status": true, + "acl": true, + "arn": true, + "bucket_domain_name": true, + "bucket_prefix": true, + "bucket_regional_domain_name": true, + "cors_rule": true, + "grant": true, + "hosted_zone_id": true, + "id": true, + "lifecycle_rule": true, + "logging": true, + "object_lock_configuration": true, + "object_lock_enabled": true, + "policy": true, + "region": true, + "replication_configuration": true, + "request_payer": true, + "server_side_encryption_configuration": true, + "tags_all": true, + "versioning": true, + "website": true, + "website_domain": true, + "website_endpoint": true + }, + "before_sensitive": false, + "after_sensitive": { + "cors_rule": [], + "grant": [], + "lifecycle_rule": [], + "logging": [], + "object_lock_configuration": [], + "replication_configuration": [], + "server_side_encryption_configuration": [], + "tags_all": {}, + "versioning": [], + "website": [] + } + } + }, + { + "address": "aws_s3_bucket_public_access_block.efrat-env-var-test", + "mode": "managed", + "type": "aws_s3_bucket_public_access_block", + "name": "efrat-env-var-test", + "provider_name": "registry.terraform.io/hashicorp/aws", + "change": { + "actions": [ + "create" + ], + "before": null, + "after": { + "block_public_acls": false, + "block_public_policy": true, + "ignore_public_acls": false, + "restrict_public_buckets": true + }, + "after_unknown": { + "bucket": true, + "id": true + }, + "before_sensitive": false, + "after_sensitive": {} + } + } + ], + "configuration": { + "provider_config": { + "aws": { + "name": "aws", + "full_name": "registry.terraform.io/hashicorp/aws", + "expressions": { + "profile": { + "constant_value": "efrat" + }, + "region": { + "constant_value": "us-east-1" + } + } + } + }, + "root_module": { + "resources": [ + { + "address": "aws_s3_bucket.efrat-env-var-test", + "mode": "managed", + "type": "aws_s3_bucket", + "name": "efrat-env-var-test", + "provider_config_key": "aws", + "expressions": { + "bucket": { + "constant_value": "efrat-env-var-test" + } + }, + "schema_version": 0 + }, + { + "address": "aws_s3_bucket_public_access_block.efrat-env-var-test", + "mode": "managed", + "type": "aws_s3_bucket_public_access_block", + "name": "efrat-env-var-test", + "provider_config_key": "aws", + "expressions": { + "block_public_acls": { + "references": [ + "var.IT_IS_FALSE" + ] + }, + "block_public_policy": { + "constant_value": true + }, + "bucket": { + "references": [ + "aws_s3_bucket.efrat-env-var-test.id", + "aws_s3_bucket.efrat-env-var-test" + ] + }, + "ignore_public_acls": { + "constant_value": false + }, + "restrict_public_buckets": { + "constant_value": true + } + }, + "schema_version": 0 + } + ], + "variables": { + "IT_IS_FALSE": { + "description": "This is an example input variable using env variables." + } + } + } + }, + "relevant_attributes": [ + { + "resource": "aws_s3_bucket.efrat-env-var-test", + "attribute": [ + "id" + ] + } + ], + "timestamp": "2023-07-31T17:54:18Z" +} \ No newline at end of file diff --git a/tests/test_files/tf_content_generator_files/tfplan-no-op-example/tf_content.txt b/tests/test_files/tf_content_generator_files/tfplan-no-op-example/tf_content.txt new file mode 100644 index 00000000..be9b8bca --- /dev/null +++ b/tests/test_files/tf_content_generator_files/tfplan-no-op-example/tf_content.txt @@ -0,0 +1,122 @@ +resource "aws_codebuild_project" "terra_ci" { + arn = "arn:aws:codebuild:eu-west-1:719261439472:project/terra-ci-runner" + artifacts = [{"artifact_identifier": "", "encryption_disabled": false, "location": "terra-ci-artifacts-eu-west-1-000002", "name": "terra-ci-runner", "namespace_type": "NONE", "override_artifact_name": false, "packaging": "NONE", "path": "", "type": "S3"}] + badge_enabled = false + badge_url = "" + build_timeout = 10 + cache = [{"location": "", "modes": [], "type": "NO_CACHE"}] + description = "Deploy environment configuration" + encryption_key = "arn:aws:kms:eu-west-1:719261439472:alias/aws/s3" + environment = [{"certificate": "", "compute_type": "BUILD_GENERAL1_SMALL", "environment_variable": [], "image": "aws/codebuild/amazonlinux2-x86_64-standard:2.0", "image_pull_credentials_type": "CODEBUILD", "privileged_mode": false, "registry_credential": [], "type": "LINUX_CONTAINER"}] + id = "arn:aws:codebuild:eu-west-1:719261439472:project/terra-ci-runner" + logs_config = [{"cloudwatch_logs": [{"group_name": "", "status": "ENABLED", "stream_name": ""}], "s3_logs": [{"encryption_disabled": false, "location": "", "status": "DISABLED"}]}] + name = "terra-ci-runner" + queued_timeout = 480 + secondary_artifacts = [] + secondary_sources = [] + service_role = "arn:aws:iam::719261439472:role/terra_ci_job" + source = [{"auth": [], "buildspec": "version: 0.2\nphases:\n install:\n commands:\n - make install_tools\n build:\n commands:\n - make plan_local resource=$TERRA_CI_RESOURCE\nartifacts:\n files:\n - ./tfplan\n name: $TERRA_CI_BUILD_NAME\n\n", "git_clone_depth": 1, "git_submodules_config": [], "insecure_ssl": false, "location": "https://github.com/p0tr3c-terraform/terra-ci-single-account.git", "report_build_status": false, "type": "GITHUB"}] + source_version = "" + tags = {} + vpc_config = [] +} + +resource "aws_iam_role" "terra_ci_job" { + arn = "arn:aws:iam::719261439472:role/terra_ci_job" + assume_role_policy = "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"codebuild.amazonaws.com\"},\"Action\":\"sts:AssumeRole\"}]}" + create_date = "2021-05-01T15:08:15Z" + description = "" + force_detach_policies = false + id = "terra_ci_job" + inline_policy = [{"name": "terraform-20210501150816628700000001", "policy": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Resource\": \"arn:aws:iam::719261439472:role/ci\"\n },\n {\n \"Effect\": \"Allow\",\n \"Resource\": [\n \"*\"\n ],\n \"Action\": [\n \"logs:CreateLogGroup\",\n \"logs:CreateLogStream\",\n \"logs:PutLogEvents\"\n ]\n },\n {\n \"Effect\": \"Allow\",\n \"Resource\": [\n \"arn:aws:s3:::terra-ci-artifacts-eu-west-1-000002\",\n \"arn:aws:s3:::terra-ci-artifacts-eu-west-1-000002/*\"\n ],\n \"Action\": [\n \"s3:ListBucket\",\n \"s3:*Object\"\n ]\n }\n ]\n}\n"}] + managed_policy_arns = ["arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryPowerUser"] + max_session_duration = 3600 + name = "terra_ci_job" + name_prefix = null + path = "/" + permissions_boundary = null + tags = {} + unique_id = "AROA2O52SSXYL7LBSM733" +} + +resource "aws_iam_role" "terra_ci_runner" { + arn = "arn:aws:iam::719261439472:role/terra_ci_runner" + assume_role_policy = "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"states.amazonaws.com\"},\"Action\":\"sts:AssumeRole\"}]}" + create_date = "2021-05-01T15:08:15Z" + description = "" + force_detach_policies = false + id = "terra_ci_runner" + inline_policy = [{"name": "terraform-20210501150825425000000003", "policy": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"codebuild:StartBuild\",\n \"codebuild:StopBuild\",\n \"codebuild:BatchGetBuilds\"\n ],\n \"Resource\": [\n \"arn:aws:codebuild:eu-west-1:719261439472:project/terra-ci-runner\"\n ]\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"events:PutTargets\",\n \"events:PutRule\",\n \"events:DescribeRule\"\n ],\n \"Resource\": [\n \"arn:aws:events:eu-west-1:719261439472:rule/StepFunctionsGetEventForCodeBuildStartBuildRule\"\n ]\n }\n ]\n}\n"}] + managed_policy_arns = [] + max_session_duration = 3600 + name = "terra_ci_runner" + name_prefix = null + path = "/" + permissions_boundary = null + tags = {} + unique_id = "AROA2O52SSXYDBYYTG4OB" +} + +resource "aws_iam_role_policy" "terra_ci_job" { + id = "terra_ci_job:terraform-20210501150816628700000001" + name = "terraform-20210501150816628700000001" + name_prefix = null + policy = "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Resource\": \"arn:aws:iam::719261439472:role/ci\"\n },\n {\n \"Effect\": \"Allow\",\n \"Resource\": [\n \"*\"\n ],\n \"Action\": [\n \"logs:CreateLogGroup\",\n \"logs:CreateLogStream\",\n \"logs:PutLogEvents\"\n ]\n },\n {\n \"Effect\": \"Allow\",\n \"Resource\": [\n \"arn:aws:s3:::terra-ci-artifacts-eu-west-1-000002\",\n \"arn:aws:s3:::terra-ci-artifacts-eu-west-1-000002/*\"\n ],\n \"Action\": [\n \"s3:ListBucket\",\n \"s3:*Object\"\n ]\n }\n ]\n}\n" + role = "terra_ci_job" +} + +resource "aws_iam_role_policy" "terra_ci_runner" { + id = "terra_ci_runner:terraform-20210501150825425000000003" + name = "terraform-20210501150825425000000003" + name_prefix = null + policy = "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"codebuild:StartBuild\",\n \"codebuild:StopBuild\",\n \"codebuild:BatchGetBuilds\"\n ],\n \"Resource\": [\n \"arn:aws:codebuild:eu-west-1:719261439472:project/terra-ci-runner\"\n ]\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"events:PutTargets\",\n \"events:PutRule\",\n \"events:DescribeRule\"\n ],\n \"Resource\": [\n \"arn:aws:events:eu-west-1:719261439472:rule/StepFunctionsGetEventForCodeBuildStartBuildRule\"\n ]\n }\n ]\n}\n" + role = "terra_ci_runner" +} + +resource "aws_iam_role_policy_attachment" "terra_ci_job_ecr_access" { + id = "terra_ci_job-20210501150817089800000002" + policy_arn = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryPowerUser" + role = "terra_ci_job" +} + +resource "aws_s3_bucket" "terra_ci" { + acceleration_status = "" + acl = "private" + arn = "arn:aws:s3:::terra-ci-artifacts-eu-west-1-000002" + bucket = "terra-ci-artifacts-eu-west-1-000002" + bucket_domain_name = "terra-ci-artifacts-eu-west-1-000002.s3.amazonaws.com" + bucket_prefix = null + bucket_regional_domain_name = "terra-ci-artifacts-eu-west-1-000002.s3.eu-west-1.amazonaws.com" + cors_rule = [] + force_destroy = false + grant = [] + hosted_zone_id = "Z1BKCTXD74EZPE" + id = "terra-ci-artifacts-eu-west-1-000002" + lifecycle_rule = [] + logging = [] + object_lock_configuration = [] + policy = null + region = "eu-west-1" + replication_configuration = [] + request_payer = "BucketOwner" + server_side_encryption_configuration = [{"rule": [{"apply_server_side_encryption_by_default": [{"kms_master_key_id": "", "sse_algorithm": "aws:kms"}], "bucket_key_enabled": false}]}] + tags = {} + versioning = [{"enabled": false, "mfa_delete": false}] + website = [] + website_domain = null + website_endpoint = null +} + +resource "aws_sfn_state_machine" "terra_ci_runner" { + arn = "arn:aws:states:eu-west-1:719261439472:stateMachine:terra-ci-runner" + creation_date = "2021-05-01T15:09:28Z" + definition = "{\n \"Comment\": \"Run Terragrunt Jobs\",\n \"StartAt\": \"OnBranch?\",\n \"States\": {\n \"OnBranch?\": {\n \"Type\": \"Choice\",\n \"Choices\": [\n {\n \"Variable\": \"$.build.sourceversion\",\n \"IsPresent\": true,\n \"Next\": \"PlanBranch\"\n }\n ],\n \"Default\": \"Plan\"\n },\n \"Plan\": {\n \"Type\": \"Task\",\n \"Resource\": \"arn:aws:states:::codebuild:startBuild.sync\",\n \"Parameters\": {\n \"ProjectName\": \"terra-ci-runner\",\n \"EnvironmentVariablesOverride\": [\n {\n \"Name\": \"TERRA_CI_BUILD_NAME\",\n \"Value.$\": \"$$.Execution.Name\"\n },\n {\n \"Name\": \"TERRA_CI_RESOURCE\",\n \"Value.$\": \"$.build.environment.terra_ci_resource\"\n }\n ]\n },\n \"End\": true\n },\n \"PlanBranch\": {\n \"Type\": \"Task\",\n \"Resource\": \"arn:aws:states:::codebuild:startBuild.sync\",\n \"Parameters\": {\n \"ProjectName\": \"terra-ci-runner\",\n \"SourceVersion.$\": \"$.build.sourceversion\",\n \"EnvironmentVariablesOverride\": [\n {\n \"Name\": \"TERRA_CI_RESOURCE\",\n \"Value.$\": \"$.build.environment.terra_ci_resource\"\n }\n ]\n },\n \"End\": true\n }\n }\n}\n" + id = "arn:aws:states:eu-west-1:719261439472:stateMachine:terra-ci-runner" + logging_configuration = [{"include_execution_data": false, "level": "OFF", "log_destination": ""}] + name = "terra-ci-runner" + role_arn = "arn:aws:iam::719261439472:role/terra_ci_runner" + status = "ACTIVE" + tags = {} + type = "STANDARD" +} + diff --git a/tests/test_files/tf_content_generator_files/tfplan-no-op-example/tfplan.json b/tests/test_files/tf_content_generator_files/tfplan-no-op-example/tfplan.json new file mode 100644 index 00000000..e31a7ea5 --- /dev/null +++ b/tests/test_files/tf_content_generator_files/tfplan-no-op-example/tfplan.json @@ -0,0 +1,1634 @@ +{ + "comment-for-reader": "THIS TERRAFORM FILE REPRESENTS A REAL TF-PLAN OUTPUT WHICH HAS VULNERABILITES IN IT'S CURRENT STATE, BUT NOT IN PROPOSED RESOURCE CHANGES", + "format_version": "0.1", + "terraform_version": "0.15.0", + "variables": { + "aws_region": { + "value": "eu-west-1" + }, + "repo_url": { + "value": "https://github.com/p0tr3c-terraform/terra-ci-single-account.git" + }, + "serial_number": { + "value": "000002" + } + }, + "planned_values": { + "root_module": { + "resources": [ + { + "address": "aws_codebuild_project.terra_ci", + "mode": "managed", + "type": "aws_codebuild_project", + "name": "terra_ci", + "provider_name": "registry.terraform.io/hashicorp/aws", + "schema_version": 0, + "values": { + "arn": "arn:aws:codebuild:eu-west-1:719261439472:project/terra-ci-runner", + "artifacts": [ + { + "artifact_identifier": "", + "encryption_disabled": false, + "location": "terra-ci-artifacts-eu-west-1-000002", + "name": "terra-ci-runner", + "namespace_type": "NONE", + "override_artifact_name": false, + "packaging": "NONE", + "path": "", + "type": "S3" + } + ], + "badge_enabled": false, + "badge_url": "", + "build_timeout": 10, + "cache": [ + { + "location": "", + "modes": [], + "type": "NO_CACHE" + } + ], + "description": "Deploy environment configuration", + "encryption_key": "arn:aws:kms:eu-west-1:719261439472:alias/aws/s3", + "environment": [ + { + "certificate": "", + "compute_type": "BUILD_GENERAL1_SMALL", + "environment_variable": [], + "image": "aws/codebuild/amazonlinux2-x86_64-standard:2.0", + "image_pull_credentials_type": "CODEBUILD", + "privileged_mode": false, + "registry_credential": [], + "type": "LINUX_CONTAINER" + } + ], + "id": "arn:aws:codebuild:eu-west-1:719261439472:project/terra-ci-runner", + "logs_config": [ + { + "cloudwatch_logs": [ + { + "group_name": "", + "status": "ENABLED", + "stream_name": "" + } + ], + "s3_logs": [ + { + "encryption_disabled": false, + "location": "", + "status": "DISABLED" + } + ] + } + ], + "name": "terra-ci-runner", + "queued_timeout": 480, + "secondary_artifacts": [], + "secondary_sources": [], + "service_role": "arn:aws:iam::719261439472:role/terra_ci_job", + "source": [ + { + "auth": [], + "buildspec": "version: 0.2\nphases:\n install:\n commands:\n - make install_tools\n build:\n commands:\n - make plan_local resource=$TERRA_CI_RESOURCE\nartifacts:\n files:\n - ./tfplan\n name: $TERRA_CI_BUILD_NAME\n\n", + "git_clone_depth": 1, + "git_submodules_config": [], + "insecure_ssl": false, + "location": "https://github.com/p0tr3c-terraform/terra-ci-single-account.git", + "report_build_status": false, + "type": "GITHUB" + } + ], + "source_version": "", + "tags": {}, + "vpc_config": [] + } + }, + { + "address": "aws_iam_role.terra_ci_job", + "mode": "managed", + "type": "aws_iam_role", + "name": "terra_ci_job", + "provider_name": "registry.terraform.io/hashicorp/aws", + "schema_version": 0, + "values": { + "arn": "arn:aws:iam::719261439472:role/terra_ci_job", + "assume_role_policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"codebuild.amazonaws.com\"},\"Action\":\"sts:AssumeRole\"}]}", + "create_date": "2021-05-01T15:08:15Z", + "description": "", + "force_detach_policies": false, + "id": "terra_ci_job", + "inline_policy": [ + { + "name": "terraform-20210501150816628700000001", + "policy": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Resource\": \"arn:aws:iam::719261439472:role/ci\"\n },\n {\n \"Effect\": \"Allow\",\n \"Resource\": [\n \"*\"\n ],\n \"Action\": [\n \"logs:CreateLogGroup\",\n \"logs:CreateLogStream\",\n \"logs:PutLogEvents\"\n ]\n },\n {\n \"Effect\": \"Allow\",\n \"Resource\": [\n \"arn:aws:s3:::terra-ci-artifacts-eu-west-1-000002\",\n \"arn:aws:s3:::terra-ci-artifacts-eu-west-1-000002/*\"\n ],\n \"Action\": [\n \"s3:ListBucket\",\n \"s3:*Object\"\n ]\n }\n ]\n}\n" + } + ], + "managed_policy_arns": [ + "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryPowerUser" + ], + "max_session_duration": 3600, + "name": "terra_ci_job", + "name_prefix": null, + "path": "/", + "permissions_boundary": null, + "tags": {}, + "unique_id": "AROA2O52SSXYL7LBSM733" + } + }, + { + "address": "aws_iam_role.terra_ci_runner", + "mode": "managed", + "type": "aws_iam_role", + "name": "terra_ci_runner", + "provider_name": "registry.terraform.io/hashicorp/aws", + "schema_version": 0, + "values": { + "arn": "arn:aws:iam::719261439472:role/terra_ci_runner", + "assume_role_policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"states.amazonaws.com\"},\"Action\":\"sts:AssumeRole\"}]}", + "create_date": "2021-05-01T15:08:15Z", + "description": "", + "force_detach_policies": false, + "id": "terra_ci_runner", + "inline_policy": [ + { + "name": "terraform-20210501150825425000000003", + "policy": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"codebuild:StartBuild\",\n \"codebuild:StopBuild\",\n \"codebuild:BatchGetBuilds\"\n ],\n \"Resource\": [\n \"arn:aws:codebuild:eu-west-1:719261439472:project/terra-ci-runner\"\n ]\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"events:PutTargets\",\n \"events:PutRule\",\n \"events:DescribeRule\"\n ],\n \"Resource\": [\n \"arn:aws:events:eu-west-1:719261439472:rule/StepFunctionsGetEventForCodeBuildStartBuildRule\"\n ]\n }\n ]\n}\n" + } + ], + "managed_policy_arns": [], + "max_session_duration": 3600, + "name": "terra_ci_runner", + "name_prefix": null, + "path": "/", + "permissions_boundary": null, + "tags": {}, + "unique_id": "AROA2O52SSXYDBYYTG4OB" + } + }, + { + "address": "aws_iam_role_policy.terra_ci_job", + "mode": "managed", + "type": "aws_iam_role_policy", + "name": "terra_ci_job", + "provider_name": "registry.terraform.io/hashicorp/aws", + "schema_version": 0, + "values": { + "id": "terra_ci_job:terraform-20210501150816628700000001", + "name": "terraform-20210501150816628700000001", + "name_prefix": null, + "policy": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Resource\": \"arn:aws:iam::719261439472:role/ci\"\n },\n {\n \"Effect\": \"Allow\",\n \"Resource\": [\n \"*\"\n ],\n \"Action\": [\n \"logs:CreateLogGroup\",\n \"logs:CreateLogStream\",\n \"logs:PutLogEvents\"\n ]\n },\n {\n \"Effect\": \"Allow\",\n \"Resource\": [\n \"arn:aws:s3:::terra-ci-artifacts-eu-west-1-000002\",\n \"arn:aws:s3:::terra-ci-artifacts-eu-west-1-000002/*\"\n ],\n \"Action\": [\n \"s3:ListBucket\",\n \"s3:*Object\"\n ]\n }\n ]\n}\n", + "role": "terra_ci_job" + } + }, + { + "address": "aws_iam_role_policy.terra_ci_runner", + "mode": "managed", + "type": "aws_iam_role_policy", + "name": "terra_ci_runner", + "provider_name": "registry.terraform.io/hashicorp/aws", + "schema_version": 0, + "values": { + "id": "terra_ci_runner:terraform-20210501150825425000000003", + "name": "terraform-20210501150825425000000003", + "name_prefix": null, + "policy": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"codebuild:StartBuild\",\n \"codebuild:StopBuild\",\n \"codebuild:BatchGetBuilds\"\n ],\n \"Resource\": [\n \"arn:aws:codebuild:eu-west-1:719261439472:project/terra-ci-runner\"\n ]\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"events:PutTargets\",\n \"events:PutRule\",\n \"events:DescribeRule\"\n ],\n \"Resource\": [\n \"arn:aws:events:eu-west-1:719261439472:rule/StepFunctionsGetEventForCodeBuildStartBuildRule\"\n ]\n }\n ]\n}\n", + "role": "terra_ci_runner" + } + }, + { + "address": "aws_iam_role_policy_attachment.terra_ci_job_ecr_access", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "terra_ci_job_ecr_access", + "provider_name": "registry.terraform.io/hashicorp/aws", + "schema_version": 0, + "values": { + "id": "terra_ci_job-20210501150817089800000002", + "policy_arn": "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryPowerUser", + "role": "terra_ci_job" + } + }, + { + "address": "aws_s3_bucket.terra_ci", + "mode": "managed", + "type": "aws_s3_bucket", + "name": "terra_ci", + "provider_name": "registry.terraform.io/hashicorp/aws", + "schema_version": 0, + "values": { + "acceleration_status": "", + "acl": "private", + "arn": "arn:aws:s3:::terra-ci-artifacts-eu-west-1-000002", + "bucket": "terra-ci-artifacts-eu-west-1-000002", + "bucket_domain_name": "terra-ci-artifacts-eu-west-1-000002.s3.amazonaws.com", + "bucket_prefix": null, + "bucket_regional_domain_name": "terra-ci-artifacts-eu-west-1-000002.s3.eu-west-1.amazonaws.com", + "cors_rule": [], + "force_destroy": false, + "grant": [], + "hosted_zone_id": "Z1BKCTXD74EZPE", + "id": "terra-ci-artifacts-eu-west-1-000002", + "lifecycle_rule": [], + "logging": [], + "object_lock_configuration": [], + "policy": null, + "region": "eu-west-1", + "replication_configuration": [], + "request_payer": "BucketOwner", + "server_side_encryption_configuration": [ + { + "rule": [ + { + "apply_server_side_encryption_by_default": [ + { + "kms_master_key_id": "", + "sse_algorithm": "aws:kms" + } + ], + "bucket_key_enabled": false + } + ] + } + ], + "tags": {}, + "versioning": [ + { + "enabled": false, + "mfa_delete": false + } + ], + "website": [], + "website_domain": null, + "website_endpoint": null + } + }, + { + "address": "aws_sfn_state_machine.terra_ci_runner", + "mode": "managed", + "type": "aws_sfn_state_machine", + "name": "terra_ci_runner", + "provider_name": "registry.terraform.io/hashicorp/aws", + "schema_version": 0, + "values": { + "arn": "arn:aws:states:eu-west-1:719261439472:stateMachine:terra-ci-runner", + "creation_date": "2021-05-01T15:09:28Z", + "definition": "{\n \"Comment\": \"Run Terragrunt Jobs\",\n \"StartAt\": \"OnBranch?\",\n \"States\": {\n \"OnBranch?\": {\n \"Type\": \"Choice\",\n \"Choices\": [\n {\n \"Variable\": \"$.build.sourceversion\",\n \"IsPresent\": true,\n \"Next\": \"PlanBranch\"\n }\n ],\n \"Default\": \"Plan\"\n },\n \"Plan\": {\n \"Type\": \"Task\",\n \"Resource\": \"arn:aws:states:::codebuild:startBuild.sync\",\n \"Parameters\": {\n \"ProjectName\": \"terra-ci-runner\",\n \"EnvironmentVariablesOverride\": [\n {\n \"Name\": \"TERRA_CI_BUILD_NAME\",\n \"Value.$\": \"$$.Execution.Name\"\n },\n {\n \"Name\": \"TERRA_CI_RESOURCE\",\n \"Value.$\": \"$.build.environment.terra_ci_resource\"\n }\n ]\n },\n \"End\": true\n },\n \"PlanBranch\": {\n \"Type\": \"Task\",\n \"Resource\": \"arn:aws:states:::codebuild:startBuild.sync\",\n \"Parameters\": {\n \"ProjectName\": \"terra-ci-runner\",\n \"SourceVersion.$\": \"$.build.sourceversion\",\n \"EnvironmentVariablesOverride\": [\n {\n \"Name\": \"TERRA_CI_RESOURCE\",\n \"Value.$\": \"$.build.environment.terra_ci_resource\"\n }\n ]\n },\n \"End\": true\n }\n }\n}\n", + "id": "arn:aws:states:eu-west-1:719261439472:stateMachine:terra-ci-runner", + "logging_configuration": [ + { + "include_execution_data": false, + "level": "OFF", + "log_destination": "" + } + ], + "name": "terra-ci-runner", + "role_arn": "arn:aws:iam::719261439472:role/terra_ci_runner", + "status": "ACTIVE", + "tags": {}, + "type": "STANDARD" + } + } + ] + } + }, + "resource_changes": [ + { + "address": "aws_codebuild_project.terra_ci", + "mode": "managed", + "type": "aws_codebuild_project", + "name": "terra_ci", + "provider_name": "registry.terraform.io/hashicorp/aws", + "change": { + "actions": [ + "no-op" + ], + "before": { + "arn": "arn:aws:codebuild:eu-west-1:719261439472:project/terra-ci-runner", + "artifacts": [ + { + "artifact_identifier": "", + "encryption_disabled": false, + "location": "terra-ci-artifacts-eu-west-1-000002", + "name": "terra-ci-runner", + "namespace_type": "NONE", + "override_artifact_name": false, + "packaging": "NONE", + "path": "", + "type": "S3" + } + ], + "badge_enabled": false, + "badge_url": "", + "build_timeout": 10, + "cache": [ + { + "location": "", + "modes": [], + "type": "NO_CACHE" + } + ], + "description": "Deploy environment configuration", + "encryption_key": "arn:aws:kms:eu-west-1:719261439472:alias/aws/s3", + "environment": [ + { + "certificate": "", + "compute_type": "BUILD_GENERAL1_SMALL", + "environment_variable": [], + "image": "aws/codebuild/amazonlinux2-x86_64-standard:2.0", + "image_pull_credentials_type": "CODEBUILD", + "privileged_mode": false, + "registry_credential": [], + "type": "LINUX_CONTAINER" + } + ], + "id": "arn:aws:codebuild:eu-west-1:719261439472:project/terra-ci-runner", + "logs_config": [ + { + "cloudwatch_logs": [ + { + "group_name": "", + "status": "ENABLED", + "stream_name": "" + } + ], + "s3_logs": [ + { + "encryption_disabled": false, + "location": "", + "status": "DISABLED" + } + ] + } + ], + "name": "terra-ci-runner", + "queued_timeout": 480, + "secondary_artifacts": [], + "secondary_sources": [], + "service_role": "arn:aws:iam::719261439472:role/terra_ci_job", + "source": [ + { + "auth": [], + "buildspec": "version: 0.2\nphases:\n install:\n commands:\n - make install_tools\n build:\n commands:\n - make plan_local resource=$TERRA_CI_RESOURCE\nartifacts:\n files:\n - ./tfplan\n name: $TERRA_CI_BUILD_NAME\n\n", + "git_clone_depth": 1, + "git_submodules_config": [], + "insecure_ssl": false, + "location": "https://github.com/p0tr3c-terraform/terra-ci-single-account.git", + "report_build_status": false, + "type": "GITHUB" + } + ], + "source_version": "", + "tags": {}, + "vpc_config": [] + }, + "after": { + "arn": "arn:aws:codebuild:eu-west-1:719261439472:project/terra-ci-runner", + "artifacts": [ + { + "artifact_identifier": "", + "encryption_disabled": false, + "location": "terra-ci-artifacts-eu-west-1-000002", + "name": "terra-ci-runner", + "namespace_type": "NONE", + "override_artifact_name": false, + "packaging": "NONE", + "path": "", + "type": "S3" + } + ], + "badge_enabled": false, + "badge_url": "", + "build_timeout": 10, + "cache": [ + { + "location": "", + "modes": [], + "type": "NO_CACHE" + } + ], + "description": "Deploy environment configuration", + "encryption_key": "arn:aws:kms:eu-west-1:719261439472:alias/aws/s3", + "environment": [ + { + "certificate": "", + "compute_type": "BUILD_GENERAL1_SMALL", + "environment_variable": [], + "image": "aws/codebuild/amazonlinux2-x86_64-standard:2.0", + "image_pull_credentials_type": "CODEBUILD", + "privileged_mode": false, + "registry_credential": [], + "type": "LINUX_CONTAINER" + } + ], + "id": "arn:aws:codebuild:eu-west-1:719261439472:project/terra-ci-runner", + "logs_config": [ + { + "cloudwatch_logs": [ + { + "group_name": "", + "status": "ENABLED", + "stream_name": "" + } + ], + "s3_logs": [ + { + "encryption_disabled": false, + "location": "", + "status": "DISABLED" + } + ] + } + ], + "name": "terra-ci-runner", + "queued_timeout": 480, + "secondary_artifacts": [], + "secondary_sources": [], + "service_role": "arn:aws:iam::719261439472:role/terra_ci_job", + "source": [ + { + "auth": [], + "buildspec": "version: 0.2\nphases:\n install:\n commands:\n - make install_tools\n build:\n commands:\n - make plan_local resource=$TERRA_CI_RESOURCE\nartifacts:\n files:\n - ./tfplan\n name: $TERRA_CI_BUILD_NAME\n\n", + "git_clone_depth": 1, + "git_submodules_config": [], + "insecure_ssl": false, + "location": "https://github.com/p0tr3c-terraform/terra-ci-single-account.git", + "report_build_status": false, + "type": "GITHUB" + } + ], + "source_version": "", + "tags": {}, + "vpc_config": [] + }, + "after_unknown": {}, + "before_sensitive": { + "artifacts": [ + {} + ], + "cache": [ + { + "modes": [] + } + ], + "environment": [ + { + "environment_variable": [], + "registry_credential": [] + } + ], + "logs_config": [ + { + "cloudwatch_logs": [ + {} + ], + "s3_logs": [ + {} + ] + } + ], + "secondary_artifacts": [], + "secondary_sources": [], + "source": [ + { + "auth": [], + "git_submodules_config": [] + } + ], + "tags": {}, + "vpc_config": [] + }, + "after_sensitive": { + "artifacts": [ + {} + ], + "cache": [ + { + "modes": [] + } + ], + "environment": [ + { + "environment_variable": [], + "registry_credential": [] + } + ], + "logs_config": [ + { + "cloudwatch_logs": [ + {} + ], + "s3_logs": [ + {} + ] + } + ], + "secondary_artifacts": [], + "secondary_sources": [], + "source": [ + { + "auth": [], + "git_submodules_config": [] + } + ], + "tags": {}, + "vpc_config": [] + } + } + }, + { + "address": "aws_iam_role.terra_ci_job", + "mode": "managed", + "type": "aws_iam_role", + "name": "terra_ci_job", + "provider_name": "registry.terraform.io/hashicorp/aws", + "change": { + "actions": [ + "no-op" + ], + "before": { + "arn": "arn:aws:iam::719261439472:role/terra_ci_job", + "assume_role_policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"codebuild.amazonaws.com\"},\"Action\":\"sts:AssumeRole\"}]}", + "create_date": "2021-05-01T15:08:15Z", + "description": "", + "force_detach_policies": false, + "id": "terra_ci_job", + "inline_policy": [ + { + "name": "terraform-20210501150816628700000001", + "policy": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Resource\": \"arn:aws:iam::719261439472:role/ci\"\n },\n {\n \"Effect\": \"Allow\",\n \"Resource\": [\n \"*\"\n ],\n \"Action\": [\n \"logs:CreateLogGroup\",\n \"logs:CreateLogStream\",\n \"logs:PutLogEvents\"\n ]\n },\n {\n \"Effect\": \"Allow\",\n \"Resource\": [\n \"arn:aws:s3:::terra-ci-artifacts-eu-west-1-000002\",\n \"arn:aws:s3:::terra-ci-artifacts-eu-west-1-000002/*\"\n ],\n \"Action\": [\n \"s3:ListBucket\",\n \"s3:*Object\"\n ]\n }\n ]\n}\n" + } + ], + "managed_policy_arns": [ + "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryPowerUser" + ], + "max_session_duration": 3600, + "name": "terra_ci_job", + "name_prefix": null, + "path": "/", + "permissions_boundary": null, + "tags": {}, + "unique_id": "AROA2O52SSXYL7LBSM733" + }, + "after": { + "arn": "arn:aws:iam::719261439472:role/terra_ci_job", + "assume_role_policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"codebuild.amazonaws.com\"},\"Action\":\"sts:AssumeRole\"}]}", + "create_date": "2021-05-01T15:08:15Z", + "description": "", + "force_detach_policies": false, + "id": "terra_ci_job", + "inline_policy": [ + { + "name": "terraform-20210501150816628700000001", + "policy": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Resource\": \"arn:aws:iam::719261439472:role/ci\"\n },\n {\n \"Effect\": \"Allow\",\n \"Resource\": [\n \"*\"\n ],\n \"Action\": [\n \"logs:CreateLogGroup\",\n \"logs:CreateLogStream\",\n \"logs:PutLogEvents\"\n ]\n },\n {\n \"Effect\": \"Allow\",\n \"Resource\": [\n \"arn:aws:s3:::terra-ci-artifacts-eu-west-1-000002\",\n \"arn:aws:s3:::terra-ci-artifacts-eu-west-1-000002/*\"\n ],\n \"Action\": [\n \"s3:ListBucket\",\n \"s3:*Object\"\n ]\n }\n ]\n}\n" + } + ], + "managed_policy_arns": [ + "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryPowerUser" + ], + "max_session_duration": 3600, + "name": "terra_ci_job", + "name_prefix": null, + "path": "/", + "permissions_boundary": null, + "tags": {}, + "unique_id": "AROA2O52SSXYL7LBSM733" + }, + "after_unknown": {}, + "before_sensitive": { + "inline_policy": [ + {} + ], + "managed_policy_arns": [ + false + ], + "tags": {} + }, + "after_sensitive": { + "inline_policy": [ + {} + ], + "managed_policy_arns": [ + false + ], + "tags": {} + } + } + }, + { + "address": "aws_iam_role.terra_ci_runner", + "mode": "managed", + "type": "aws_iam_role", + "name": "terra_ci_runner", + "provider_name": "registry.terraform.io/hashicorp/aws", + "change": { + "actions": [ + "no-op" + ], + "before": { + "arn": "arn:aws:iam::719261439472:role/terra_ci_runner", + "assume_role_policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"states.amazonaws.com\"},\"Action\":\"sts:AssumeRole\"}]}", + "create_date": "2021-05-01T15:08:15Z", + "description": "", + "force_detach_policies": false, + "id": "terra_ci_runner", + "inline_policy": [ + { + "name": "terraform-20210501150825425000000003", + "policy": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"codebuild:StartBuild\",\n \"codebuild:StopBuild\",\n \"codebuild:BatchGetBuilds\"\n ],\n \"Resource\": [\n \"arn:aws:codebuild:eu-west-1:719261439472:project/terra-ci-runner\"\n ]\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"events:PutTargets\",\n \"events:PutRule\",\n \"events:DescribeRule\"\n ],\n \"Resource\": [\n \"arn:aws:events:eu-west-1:719261439472:rule/StepFunctionsGetEventForCodeBuildStartBuildRule\"\n ]\n }\n ]\n}\n" + } + ], + "managed_policy_arns": [], + "max_session_duration": 3600, + "name": "terra_ci_runner", + "name_prefix": null, + "path": "/", + "permissions_boundary": null, + "tags": {}, + "unique_id": "AROA2O52SSXYDBYYTG4OB" + }, + "after": { + "arn": "arn:aws:iam::719261439472:role/terra_ci_runner", + "assume_role_policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"states.amazonaws.com\"},\"Action\":\"sts:AssumeRole\"}]}", + "create_date": "2021-05-01T15:08:15Z", + "description": "", + "force_detach_policies": false, + "id": "terra_ci_runner", + "inline_policy": [ + { + "name": "terraform-20210501150825425000000003", + "policy": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"codebuild:StartBuild\",\n \"codebuild:StopBuild\",\n \"codebuild:BatchGetBuilds\"\n ],\n \"Resource\": [\n \"arn:aws:codebuild:eu-west-1:719261439472:project/terra-ci-runner\"\n ]\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"events:PutTargets\",\n \"events:PutRule\",\n \"events:DescribeRule\"\n ],\n \"Resource\": [\n \"arn:aws:events:eu-west-1:719261439472:rule/StepFunctionsGetEventForCodeBuildStartBuildRule\"\n ]\n }\n ]\n}\n" + } + ], + "managed_policy_arns": [], + "max_session_duration": 3600, + "name": "terra_ci_runner", + "name_prefix": null, + "path": "/", + "permissions_boundary": null, + "tags": {}, + "unique_id": "AROA2O52SSXYDBYYTG4OB" + }, + "after_unknown": {}, + "before_sensitive": { + "inline_policy": [ + {} + ], + "managed_policy_arns": [], + "tags": {} + }, + "after_sensitive": { + "inline_policy": [ + {} + ], + "managed_policy_arns": [], + "tags": {} + } + } + }, + { + "address": "aws_iam_role_policy.terra_ci_job", + "mode": "managed", + "type": "aws_iam_role_policy", + "name": "terra_ci_job", + "provider_name": "registry.terraform.io/hashicorp/aws", + "change": { + "actions": [ + "no-op" + ], + "before": { + "id": "terra_ci_job:terraform-20210501150816628700000001", + "name": "terraform-20210501150816628700000001", + "name_prefix": null, + "policy": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Resource\": \"arn:aws:iam::719261439472:role/ci\"\n },\n {\n \"Effect\": \"Allow\",\n \"Resource\": [\n \"*\"\n ],\n \"Action\": [\n \"logs:CreateLogGroup\",\n \"logs:CreateLogStream\",\n \"logs:PutLogEvents\"\n ]\n },\n {\n \"Effect\": \"Allow\",\n \"Resource\": [\n \"arn:aws:s3:::terra-ci-artifacts-eu-west-1-000002\",\n \"arn:aws:s3:::terra-ci-artifacts-eu-west-1-000002/*\"\n ],\n \"Action\": [\n \"s3:ListBucket\",\n \"s3:*Object\"\n ]\n }\n ]\n}\n", + "role": "terra_ci_job" + }, + "after": { + "id": "terra_ci_job:terraform-20210501150816628700000001", + "name": "terraform-20210501150816628700000001", + "name_prefix": null, + "policy": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Resource\": \"arn:aws:iam::719261439472:role/ci\"\n },\n {\n \"Effect\": \"Allow\",\n \"Resource\": [\n \"*\"\n ],\n \"Action\": [\n \"logs:CreateLogGroup\",\n \"logs:CreateLogStream\",\n \"logs:PutLogEvents\"\n ]\n },\n {\n \"Effect\": \"Allow\",\n \"Resource\": [\n \"arn:aws:s3:::terra-ci-artifacts-eu-west-1-000002\",\n \"arn:aws:s3:::terra-ci-artifacts-eu-west-1-000002/*\"\n ],\n \"Action\": [\n \"s3:ListBucket\",\n \"s3:*Object\"\n ]\n }\n ]\n}\n", + "role": "terra_ci_job" + }, + "after_unknown": {}, + "before_sensitive": {}, + "after_sensitive": {} + } + }, + { + "address": "aws_iam_role_policy.terra_ci_runner", + "mode": "managed", + "type": "aws_iam_role_policy", + "name": "terra_ci_runner", + "provider_name": "registry.terraform.io/hashicorp/aws", + "change": { + "actions": [ + "no-op" + ], + "before": { + "id": "terra_ci_runner:terraform-20210501150825425000000003", + "name": "terraform-20210501150825425000000003", + "name_prefix": null, + "policy": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"codebuild:StartBuild\",\n \"codebuild:StopBuild\",\n \"codebuild:BatchGetBuilds\"\n ],\n \"Resource\": [\n \"arn:aws:codebuild:eu-west-1:719261439472:project/terra-ci-runner\"\n ]\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"events:PutTargets\",\n \"events:PutRule\",\n \"events:DescribeRule\"\n ],\n \"Resource\": [\n \"arn:aws:events:eu-west-1:719261439472:rule/StepFunctionsGetEventForCodeBuildStartBuildRule\"\n ]\n }\n ]\n}\n", + "role": "terra_ci_runner" + }, + "after": { + "id": "terra_ci_runner:terraform-20210501150825425000000003", + "name": "terraform-20210501150825425000000003", + "name_prefix": null, + "policy": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"codebuild:StartBuild\",\n \"codebuild:StopBuild\",\n \"codebuild:BatchGetBuilds\"\n ],\n \"Resource\": [\n \"arn:aws:codebuild:eu-west-1:719261439472:project/terra-ci-runner\"\n ]\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"events:PutTargets\",\n \"events:PutRule\",\n \"events:DescribeRule\"\n ],\n \"Resource\": [\n \"arn:aws:events:eu-west-1:719261439472:rule/StepFunctionsGetEventForCodeBuildStartBuildRule\"\n ]\n }\n ]\n}\n", + "role": "terra_ci_runner" + }, + "after_unknown": {}, + "before_sensitive": {}, + "after_sensitive": {} + } + }, + { + "address": "aws_iam_role_policy_attachment.terra_ci_job_ecr_access", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "terra_ci_job_ecr_access", + "provider_name": "registry.terraform.io/hashicorp/aws", + "change": { + "actions": [ + "no-op" + ], + "before": { + "id": "terra_ci_job-20210501150817089800000002", + "policy_arn": "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryPowerUser", + "role": "terra_ci_job" + }, + "after": { + "id": "terra_ci_job-20210501150817089800000002", + "policy_arn": "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryPowerUser", + "role": "terra_ci_job" + }, + "after_unknown": {}, + "before_sensitive": {}, + "after_sensitive": {} + } + }, + { + "address": "aws_s3_bucket.terra_ci", + "mode": "managed", + "type": "aws_s3_bucket", + "name": "terra_ci", + "provider_name": "registry.terraform.io/hashicorp/aws", + "change": { + "actions": [ + "no-op" + ], + "before": { + "acceleration_status": "", + "acl": "private", + "arn": "arn:aws:s3:::terra-ci-artifacts-eu-west-1-000002", + "bucket": "terra-ci-artifacts-eu-west-1-000002", + "bucket_domain_name": "terra-ci-artifacts-eu-west-1-000002.s3.amazonaws.com", + "bucket_prefix": null, + "bucket_regional_domain_name": "terra-ci-artifacts-eu-west-1-000002.s3.eu-west-1.amazonaws.com", + "cors_rule": [], + "force_destroy": false, + "grant": [], + "hosted_zone_id": "Z1BKCTXD74EZPE", + "id": "terra-ci-artifacts-eu-west-1-000002", + "lifecycle_rule": [], + "logging": [], + "object_lock_configuration": [], + "policy": null, + "region": "eu-west-1", + "replication_configuration": [], + "request_payer": "BucketOwner", + "server_side_encryption_configuration": [ + { + "rule": [ + { + "apply_server_side_encryption_by_default": [ + { + "kms_master_key_id": "", + "sse_algorithm": "aws:kms" + } + ], + "bucket_key_enabled": false + } + ] + } + ], + "tags": {}, + "versioning": [ + { + "enabled": false, + "mfa_delete": false + } + ], + "website": [], + "website_domain": null, + "website_endpoint": null + }, + "after": { + "acceleration_status": "", + "acl": "private", + "arn": "arn:aws:s3:::terra-ci-artifacts-eu-west-1-000002", + "bucket": "terra-ci-artifacts-eu-west-1-000002", + "bucket_domain_name": "terra-ci-artifacts-eu-west-1-000002.s3.amazonaws.com", + "bucket_prefix": null, + "bucket_regional_domain_name": "terra-ci-artifacts-eu-west-1-000002.s3.eu-west-1.amazonaws.com", + "cors_rule": [], + "force_destroy": false, + "grant": [], + "hosted_zone_id": "Z1BKCTXD74EZPE", + "id": "terra-ci-artifacts-eu-west-1-000002", + "lifecycle_rule": [], + "logging": [], + "object_lock_configuration": [], + "policy": null, + "region": "eu-west-1", + "replication_configuration": [], + "request_payer": "BucketOwner", + "server_side_encryption_configuration": [ + { + "rule": [ + { + "apply_server_side_encryption_by_default": [ + { + "kms_master_key_id": "", + "sse_algorithm": "aws:kms" + } + ], + "bucket_key_enabled": false + } + ] + } + ], + "tags": {}, + "versioning": [ + { + "enabled": false, + "mfa_delete": false + } + ], + "website": [], + "website_domain": null, + "website_endpoint": null + }, + "after_unknown": {}, + "before_sensitive": { + "cors_rule": [], + "grant": [], + "lifecycle_rule": [], + "logging": [], + "object_lock_configuration": [], + "replication_configuration": [], + "server_side_encryption_configuration": [ + { + "rule": [ + { + "apply_server_side_encryption_by_default": [ + {} + ] + } + ] + } + ], + "tags": {}, + "versioning": [ + {} + ], + "website": [] + }, + "after_sensitive": { + "cors_rule": [], + "grant": [], + "lifecycle_rule": [], + "logging": [], + "object_lock_configuration": [], + "replication_configuration": [], + "server_side_encryption_configuration": [ + { + "rule": [ + { + "apply_server_side_encryption_by_default": [ + {} + ] + } + ] + } + ], + "tags": {}, + "versioning": [ + {} + ], + "website": [] + } + } + }, + { + "address": "aws_sfn_state_machine.terra_ci_runner", + "mode": "managed", + "type": "aws_sfn_state_machine", + "name": "terra_ci_runner", + "provider_name": "registry.terraform.io/hashicorp/aws", + "change": { + "actions": [ + "no-op" + ], + "before": { + "arn": "arn:aws:states:eu-west-1:719261439472:stateMachine:terra-ci-runner", + "creation_date": "2021-05-01T15:09:28Z", + "definition": "{\n \"Comment\": \"Run Terragrunt Jobs\",\n \"StartAt\": \"OnBranch?\",\n \"States\": {\n \"OnBranch?\": {\n \"Type\": \"Choice\",\n \"Choices\": [\n {\n \"Variable\": \"$.build.sourceversion\",\n \"IsPresent\": true,\n \"Next\": \"PlanBranch\"\n }\n ],\n \"Default\": \"Plan\"\n },\n \"Plan\": {\n \"Type\": \"Task\",\n \"Resource\": \"arn:aws:states:::codebuild:startBuild.sync\",\n \"Parameters\": {\n \"ProjectName\": \"terra-ci-runner\",\n \"EnvironmentVariablesOverride\": [\n {\n \"Name\": \"TERRA_CI_BUILD_NAME\",\n \"Value.$\": \"$$.Execution.Name\"\n },\n {\n \"Name\": \"TERRA_CI_RESOURCE\",\n \"Value.$\": \"$.build.environment.terra_ci_resource\"\n }\n ]\n },\n \"End\": true\n },\n \"PlanBranch\": {\n \"Type\": \"Task\",\n \"Resource\": \"arn:aws:states:::codebuild:startBuild.sync\",\n \"Parameters\": {\n \"ProjectName\": \"terra-ci-runner\",\n \"SourceVersion.$\": \"$.build.sourceversion\",\n \"EnvironmentVariablesOverride\": [\n {\n \"Name\": \"TERRA_CI_RESOURCE\",\n \"Value.$\": \"$.build.environment.terra_ci_resource\"\n }\n ]\n },\n \"End\": true\n }\n }\n}\n", + "id": "arn:aws:states:eu-west-1:719261439472:stateMachine:terra-ci-runner", + "logging_configuration": [ + { + "include_execution_data": false, + "level": "OFF", + "log_destination": "" + } + ], + "name": "terra-ci-runner", + "role_arn": "arn:aws:iam::719261439472:role/terra_ci_runner", + "status": "ACTIVE", + "tags": {}, + "type": "STANDARD" + }, + "after": { + "arn": "arn:aws:states:eu-west-1:719261439472:stateMachine:terra-ci-runner", + "creation_date": "2021-05-01T15:09:28Z", + "definition": "{\n \"Comment\": \"Run Terragrunt Jobs\",\n \"StartAt\": \"OnBranch?\",\n \"States\": {\n \"OnBranch?\": {\n \"Type\": \"Choice\",\n \"Choices\": [\n {\n \"Variable\": \"$.build.sourceversion\",\n \"IsPresent\": true,\n \"Next\": \"PlanBranch\"\n }\n ],\n \"Default\": \"Plan\"\n },\n \"Plan\": {\n \"Type\": \"Task\",\n \"Resource\": \"arn:aws:states:::codebuild:startBuild.sync\",\n \"Parameters\": {\n \"ProjectName\": \"terra-ci-runner\",\n \"EnvironmentVariablesOverride\": [\n {\n \"Name\": \"TERRA_CI_BUILD_NAME\",\n \"Value.$\": \"$$.Execution.Name\"\n },\n {\n \"Name\": \"TERRA_CI_RESOURCE\",\n \"Value.$\": \"$.build.environment.terra_ci_resource\"\n }\n ]\n },\n \"End\": true\n },\n \"PlanBranch\": {\n \"Type\": \"Task\",\n \"Resource\": \"arn:aws:states:::codebuild:startBuild.sync\",\n \"Parameters\": {\n \"ProjectName\": \"terra-ci-runner\",\n \"SourceVersion.$\": \"$.build.sourceversion\",\n \"EnvironmentVariablesOverride\": [\n {\n \"Name\": \"TERRA_CI_RESOURCE\",\n \"Value.$\": \"$.build.environment.terra_ci_resource\"\n }\n ]\n },\n \"End\": true\n }\n }\n}\n", + "id": "arn:aws:states:eu-west-1:719261439472:stateMachine:terra-ci-runner", + "logging_configuration": [ + { + "include_execution_data": false, + "level": "OFF", + "log_destination": "" + } + ], + "name": "terra-ci-runner", + "role_arn": "arn:aws:iam::719261439472:role/terra_ci_runner", + "status": "ACTIVE", + "tags": {}, + "type": "STANDARD" + }, + "after_unknown": {}, + "before_sensitive": { + "logging_configuration": [ + {} + ], + "tags": {} + }, + "after_sensitive": { + "logging_configuration": [ + {} + ], + "tags": {} + } + } + } + ], + "prior_state": { + "format_version": "0.1", + "terraform_version": "0.15.0", + "values": { + "root_module": { + "resources": [ + { + "address": "aws_codebuild_project.terra_ci", + "mode": "managed", + "type": "aws_codebuild_project", + "name": "terra_ci", + "provider_name": "registry.terraform.io/hashicorp/aws", + "schema_version": 0, + "values": { + "arn": "arn:aws:codebuild:eu-west-1:719261439472:project/terra-ci-runner", + "artifacts": [ + { + "artifact_identifier": "", + "encryption_disabled": false, + "location": "terra-ci-artifacts-eu-west-1-000002", + "name": "terra-ci-runner", + "namespace_type": "NONE", + "override_artifact_name": false, + "packaging": "NONE", + "path": "", + "type": "S3" + } + ], + "badge_enabled": false, + "badge_url": "", + "build_timeout": 10, + "cache": [ + { + "location": "", + "modes": [], + "type": "NO_CACHE" + } + ], + "description": "Deploy environment configuration", + "encryption_key": "arn:aws:kms:eu-west-1:719261439472:alias/aws/s3", + "environment": [ + { + "certificate": "", + "compute_type": "BUILD_GENERAL1_SMALL", + "environment_variable": [], + "image": "aws/codebuild/amazonlinux2-x86_64-standard:2.0", + "image_pull_credentials_type": "CODEBUILD", + "privileged_mode": false, + "registry_credential": [], + "type": "LINUX_CONTAINER" + } + ], + "id": "arn:aws:codebuild:eu-west-1:719261439472:project/terra-ci-runner", + "logs_config": [ + { + "cloudwatch_logs": [ + { + "group_name": "", + "status": "ENABLED", + "stream_name": "" + } + ], + "s3_logs": [ + { + "encryption_disabled": false, + "location": "", + "status": "DISABLED" + } + ] + } + ], + "name": "terra-ci-runner", + "queued_timeout": 480, + "secondary_artifacts": [], + "secondary_sources": [], + "service_role": "arn:aws:iam::719261439472:role/terra_ci_job", + "source": [ + { + "auth": [], + "buildspec": "version: 0.2\nphases:\n install:\n commands:\n - make install_tools\n build:\n commands:\n - make plan_local resource=$TERRA_CI_RESOURCE\nartifacts:\n files:\n - ./tfplan\n name: $TERRA_CI_BUILD_NAME\n\n", + "git_clone_depth": 1, + "git_submodules_config": [], + "insecure_ssl": false, + "location": "https://github.com/p0tr3c-terraform/terra-ci-single-account.git", + "report_build_status": false, + "type": "GITHUB" + } + ], + "source_version": "", + "tags": {}, + "vpc_config": [] + }, + "depends_on": [ + "aws_iam_role.terra_ci_job", + "aws_s3_bucket.terra_ci", + "data.template_file.terra_ci" + ] + }, + { + "address": "aws_iam_role.terra_ci_job", + "mode": "managed", + "type": "aws_iam_role", + "name": "terra_ci_job", + "provider_name": "registry.terraform.io/hashicorp/aws", + "schema_version": 0, + "values": { + "arn": "arn:aws:iam::719261439472:role/terra_ci_job", + "assume_role_policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"codebuild.amazonaws.com\"},\"Action\":\"sts:AssumeRole\"}]}", + "create_date": "2021-05-01T15:08:15Z", + "description": "", + "force_detach_policies": false, + "id": "terra_ci_job", + "inline_policy": [ + { + "name": "terraform-20210501150816628700000001", + "policy": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Resource\": \"arn:aws:iam::719261439472:role/ci\"\n },\n {\n \"Effect\": \"Allow\",\n \"Resource\": [\n \"*\"\n ],\n \"Action\": [\n \"logs:CreateLogGroup\",\n \"logs:CreateLogStream\",\n \"logs:PutLogEvents\"\n ]\n },\n {\n \"Effect\": \"Allow\",\n \"Resource\": [\n \"arn:aws:s3:::terra-ci-artifacts-eu-west-1-000002\",\n \"arn:aws:s3:::terra-ci-artifacts-eu-west-1-000002/*\"\n ],\n \"Action\": [\n \"s3:ListBucket\",\n \"s3:*Object\"\n ]\n }\n ]\n}\n" + } + ], + "managed_policy_arns": [ + "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryPowerUser" + ], + "max_session_duration": 3600, + "name": "terra_ci_job", + "name_prefix": null, + "path": "/", + "permissions_boundary": null, + "tags": {}, + "unique_id": "AROA2O52SSXYL7LBSM733" + } + }, + { + "address": "aws_iam_role.terra_ci_runner", + "mode": "managed", + "type": "aws_iam_role", + "name": "terra_ci_runner", + "provider_name": "registry.terraform.io/hashicorp/aws", + "schema_version": 0, + "values": { + "arn": "arn:aws:iam::719261439472:role/terra_ci_runner", + "assume_role_policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"states.amazonaws.com\"},\"Action\":\"sts:AssumeRole\"}]}", + "create_date": "2021-05-01T15:08:15Z", + "description": "", + "force_detach_policies": false, + "id": "terra_ci_runner", + "inline_policy": [ + { + "name": "terraform-20210501150825425000000003", + "policy": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"codebuild:StartBuild\",\n \"codebuild:StopBuild\",\n \"codebuild:BatchGetBuilds\"\n ],\n \"Resource\": [\n \"arn:aws:codebuild:eu-west-1:719261439472:project/terra-ci-runner\"\n ]\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"events:PutTargets\",\n \"events:PutRule\",\n \"events:DescribeRule\"\n ],\n \"Resource\": [\n \"arn:aws:events:eu-west-1:719261439472:rule/StepFunctionsGetEventForCodeBuildStartBuildRule\"\n ]\n }\n ]\n}\n" + } + ], + "managed_policy_arns": [], + "max_session_duration": 3600, + "name": "terra_ci_runner", + "name_prefix": null, + "path": "/", + "permissions_boundary": null, + "tags": {}, + "unique_id": "AROA2O52SSXYDBYYTG4OB" + } + }, + { + "address": "aws_iam_role_policy.terra_ci_job", + "mode": "managed", + "type": "aws_iam_role_policy", + "name": "terra_ci_job", + "provider_name": "registry.terraform.io/hashicorp/aws", + "schema_version": 0, + "values": { + "id": "terra_ci_job:terraform-20210501150816628700000001", + "name": "terraform-20210501150816628700000001", + "name_prefix": null, + "policy": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": \"sts:AssumeRole\",\n \"Resource\": \"arn:aws:iam::719261439472:role/ci\"\n },\n {\n \"Effect\": \"Allow\",\n \"Resource\": [\n \"*\"\n ],\n \"Action\": [\n \"logs:CreateLogGroup\",\n \"logs:CreateLogStream\",\n \"logs:PutLogEvents\"\n ]\n },\n {\n \"Effect\": \"Allow\",\n \"Resource\": [\n \"arn:aws:s3:::terra-ci-artifacts-eu-west-1-000002\",\n \"arn:aws:s3:::terra-ci-artifacts-eu-west-1-000002/*\"\n ],\n \"Action\": [\n \"s3:ListBucket\",\n \"s3:*Object\"\n ]\n }\n ]\n}\n", + "role": "terra_ci_job" + }, + "depends_on": [ + "aws_iam_role.terra_ci_job", + "aws_s3_bucket.terra_ci", + "data.aws_caller_identity.current" + ] + }, + { + "address": "aws_iam_role_policy.terra_ci_runner", + "mode": "managed", + "type": "aws_iam_role_policy", + "name": "terra_ci_runner", + "provider_name": "registry.terraform.io/hashicorp/aws", + "schema_version": 0, + "values": { + "id": "terra_ci_runner:terraform-20210501150825425000000003", + "name": "terraform-20210501150825425000000003", + "name_prefix": null, + "policy": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"codebuild:StartBuild\",\n \"codebuild:StopBuild\",\n \"codebuild:BatchGetBuilds\"\n ],\n \"Resource\": [\n \"arn:aws:codebuild:eu-west-1:719261439472:project/terra-ci-runner\"\n ]\n },\n {\n \"Effect\": \"Allow\",\n \"Action\": [\n \"events:PutTargets\",\n \"events:PutRule\",\n \"events:DescribeRule\"\n ],\n \"Resource\": [\n \"arn:aws:events:eu-west-1:719261439472:rule/StepFunctionsGetEventForCodeBuildStartBuildRule\"\n ]\n }\n ]\n}\n", + "role": "terra_ci_runner" + }, + "depends_on": [ + "data.template_file.terra_ci", + "aws_codebuild_project.terra_ci", + "aws_iam_role.terra_ci_job", + "aws_iam_role.terra_ci_runner", + "aws_s3_bucket.terra_ci", + "data.aws_caller_identity.current" + ] + }, + { + "address": "aws_iam_role_policy_attachment.terra_ci_job_ecr_access", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "terra_ci_job_ecr_access", + "provider_name": "registry.terraform.io/hashicorp/aws", + "schema_version": 0, + "values": { + "id": "terra_ci_job-20210501150817089800000002", + "policy_arn": "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryPowerUser", + "role": "terra_ci_job" + }, + "depends_on": [ + "aws_iam_role.terra_ci_job" + ] + }, + { + "address": "aws_s3_bucket.terra_ci", + "mode": "managed", + "type": "aws_s3_bucket", + "name": "terra_ci", + "provider_name": "registry.terraform.io/hashicorp/aws", + "schema_version": 0, + "values": { + "acceleration_status": "", + "acl": "private", + "arn": "arn:aws:s3:::terra-ci-artifacts-eu-west-1-000002", + "bucket": "terra-ci-artifacts-eu-west-1-000002", + "bucket_domain_name": "terra-ci-artifacts-eu-west-1-000002.s3.amazonaws.com", + "bucket_prefix": null, + "bucket_regional_domain_name": "terra-ci-artifacts-eu-west-1-000002.s3.eu-west-1.amazonaws.com", + "cors_rule": [], + "force_destroy": false, + "grant": [], + "hosted_zone_id": "Z1BKCTXD74EZPE", + "id": "terra-ci-artifacts-eu-west-1-000002", + "lifecycle_rule": [], + "logging": [], + "object_lock_configuration": [], + "policy": null, + "region": "eu-west-1", + "replication_configuration": [], + "request_payer": "BucketOwner", + "server_side_encryption_configuration": [ + { + "rule": [ + { + "apply_server_side_encryption_by_default": [ + { + "kms_master_key_id": "", + "sse_algorithm": "aws:kms" + } + ], + "bucket_key_enabled": false + } + ] + } + ], + "tags": {}, + "versioning": [ + { + "enabled": false, + "mfa_delete": false + } + ], + "website": [], + "website_domain": null, + "website_endpoint": null + } + }, + { + "address": "aws_sfn_state_machine.terra_ci_runner", + "mode": "managed", + "type": "aws_sfn_state_machine", + "name": "terra_ci_runner", + "provider_name": "registry.terraform.io/hashicorp/aws", + "schema_version": 0, + "values": { + "arn": "arn:aws:states:eu-west-1:719261439472:stateMachine:terra-ci-runner", + "creation_date": "2021-05-01T15:09:28Z", + "definition": "{\n \"Comment\": \"Run Terragrunt Jobs\",\n \"StartAt\": \"OnBranch?\",\n \"States\": {\n \"OnBranch?\": {\n \"Type\": \"Choice\",\n \"Choices\": [\n {\n \"Variable\": \"$.build.sourceversion\",\n \"IsPresent\": true,\n \"Next\": \"PlanBranch\"\n }\n ],\n \"Default\": \"Plan\"\n },\n \"Plan\": {\n \"Type\": \"Task\",\n \"Resource\": \"arn:aws:states:::codebuild:startBuild.sync\",\n \"Parameters\": {\n \"ProjectName\": \"terra-ci-runner\",\n \"EnvironmentVariablesOverride\": [\n {\n \"Name\": \"TERRA_CI_BUILD_NAME\",\n \"Value.$\": \"$$.Execution.Name\"\n },\n {\n \"Name\": \"TERRA_CI_RESOURCE\",\n \"Value.$\": \"$.build.environment.terra_ci_resource\"\n }\n ]\n },\n \"End\": true\n },\n \"PlanBranch\": {\n \"Type\": \"Task\",\n \"Resource\": \"arn:aws:states:::codebuild:startBuild.sync\",\n \"Parameters\": {\n \"ProjectName\": \"terra-ci-runner\",\n \"SourceVersion.$\": \"$.build.sourceversion\",\n \"EnvironmentVariablesOverride\": [\n {\n \"Name\": \"TERRA_CI_RESOURCE\",\n \"Value.$\": \"$.build.environment.terra_ci_resource\"\n }\n ]\n },\n \"End\": true\n }\n }\n}\n", + "id": "arn:aws:states:eu-west-1:719261439472:stateMachine:terra-ci-runner", + "logging_configuration": [ + { + "include_execution_data": false, + "level": "OFF", + "log_destination": "" + } + ], + "name": "terra-ci-runner", + "role_arn": "arn:aws:iam::719261439472:role/terra_ci_runner", + "status": "ACTIVE", + "tags": {}, + "type": "STANDARD" + }, + "depends_on": [ + "aws_codebuild_project.terra_ci", + "aws_iam_role.terra_ci_job", + "aws_iam_role.terra_ci_runner", + "aws_s3_bucket.terra_ci", + "data.template_file.terra_ci" + ] + }, + { + "address": "data.aws_caller_identity.current", + "mode": "data", + "type": "aws_caller_identity", + "name": "current", + "provider_name": "registry.terraform.io/hashicorp/aws", + "schema_version": 0, + "values": { + "account_id": "719261439472", + "arn": "arn:aws:sts::719261439472:assumed-role/ci/1620199336456840848", + "id": "719261439472", + "user_id": "AROA2O52SSXYLVFYURBIV:1620199336456840848" + } + }, + { + "address": "data.template_file.terra_ci", + "mode": "data", + "type": "template_file", + "name": "terra_ci", + "provider_name": "registry.terraform.io/hashicorp/template", + "schema_version": 0, + "values": { + "filename": null, + "id": "64e36ed71e7270140dde96fec9c89d1d55ae5a6e91f7c0be15170200dcf9481b", + "rendered": "version: 0.2\nphases:\n install:\n commands:\n - make install_tools\n build:\n commands:\n - make plan_local resource=$TERRA_CI_RESOURCE\nartifacts:\n files:\n - ./tfplan\n name: $TERRA_CI_BUILD_NAME\n\n", + "template": "version: 0.2\nphases:\n install:\n commands:\n - make install_tools\n build:\n commands:\n - make plan_local resource=$TERRA_CI_RESOURCE\nartifacts:\n files:\n - ./tfplan\n name: $TERRA_CI_BUILD_NAME\n\n", + "vars": null + } + } + ] + } + } + }, + "configuration": { + "provider_config": { + "aws": { + "name": "aws", + "expressions": { + "allowed_account_ids": { + "constant_value": [ + "719261439472" + ] + }, + "assume_role": [ + { + "role_arn": { + "constant_value": "arn:aws:iam::719261439472:role/ci" + } + } + ], + "region": { + "constant_value": "eu-west-1" + } + } + } + }, + "root_module": { + "resources": [ + { + "address": "aws_codebuild_project.terra_ci", + "mode": "managed", + "type": "aws_codebuild_project", + "name": "terra_ci", + "provider_config_key": "aws", + "expressions": { + "artifacts": [ + { + "location": { + "references": [ + "aws_s3_bucket.terra_ci" + ] + }, + "type": { + "constant_value": "S3" + } + } + ], + "build_timeout": { + "constant_value": "10" + }, + "description": { + "constant_value": "Deploy environment configuration" + }, + "environment": [ + { + "compute_type": { + "constant_value": "BUILD_GENERAL1_SMALL" + }, + "image": { + "constant_value": "aws/codebuild/amazonlinux2-x86_64-standard:2.0" + }, + "image_pull_credentials_type": { + "constant_value": "CODEBUILD" + }, + "privileged_mode": { + "constant_value": false + }, + "type": { + "constant_value": "LINUX_CONTAINER" + } + } + ], + "logs_config": [ + { + "cloudwatch_logs": [ + { + "status": { + "constant_value": "ENABLED" + } + } + ], + "s3_logs": [ + { + "encryption_disabled": { + "constant_value": false + }, + "status": { + "constant_value": "DISABLED" + } + } + ] + } + ], + "name": { + "constant_value": "terra-ci-runner" + }, + "service_role": { + "references": [ + "aws_iam_role.terra_ci_job" + ] + }, + "source": [ + { + "buildspec": { + "references": [ + "data.template_file.terra_ci" + ] + }, + "git_clone_depth": { + "constant_value": 1 + }, + "insecure_ssl": { + "constant_value": false + }, + "location": { + "references": [ + "var.repo_url" + ] + }, + "report_build_status": { + "constant_value": false + }, + "type": { + "constant_value": "GITHUB" + } + } + ] + }, + "schema_version": 0 + }, + { + "address": "aws_iam_role.terra_ci_job", + "mode": "managed", + "type": "aws_iam_role", + "name": "terra_ci_job", + "provider_config_key": "aws", + "expressions": { + "assume_role_policy": { + "constant_value": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Principal\": {\n \"Service\": \"codebuild.amazonaws.com\"\n },\n \"Action\": \"sts:AssumeRole\"\n }\n ]\n}\n" + }, + "name": { + "constant_value": "terra_ci_job" + } + }, + "schema_version": 0 + }, + { + "address": "aws_iam_role.terra_ci_runner", + "mode": "managed", + "type": "aws_iam_role", + "name": "terra_ci_runner", + "provider_config_key": "aws", + "expressions": { + "assume_role_policy": { + "constant_value": "{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Effect\": \"Allow\",\n \"Principal\": {\n \"Service\": \"states.amazonaws.com\"\n },\n \"Action\": \"sts:AssumeRole\"\n }\n ]\n}\n" + }, + "name": { + "constant_value": "terra_ci_runner" + } + }, + "schema_version": 0 + }, + { + "address": "aws_iam_role_policy.terra_ci_job", + "mode": "managed", + "type": "aws_iam_role_policy", + "name": "terra_ci_job", + "provider_config_key": "aws", + "expressions": { + "policy": { + "references": [ + "data.aws_caller_identity.current", + "aws_s3_bucket.terra_ci", + "aws_s3_bucket.terra_ci" + ] + }, + "role": { + "references": [ + "aws_iam_role.terra_ci_job" + ] + } + }, + "schema_version": 0 + }, + { + "address": "aws_iam_role_policy.terra_ci_runner", + "mode": "managed", + "type": "aws_iam_role_policy", + "name": "terra_ci_runner", + "provider_config_key": "aws", + "expressions": { + "policy": { + "references": [ + "aws_codebuild_project.terra_ci", + "var.aws_region", + "data.aws_caller_identity.current" + ] + }, + "role": { + "references": [ + "aws_iam_role.terra_ci_runner" + ] + } + }, + "schema_version": 0 + }, + { + "address": "aws_iam_role_policy_attachment.terra_ci_job_ecr_access", + "mode": "managed", + "type": "aws_iam_role_policy_attachment", + "name": "terra_ci_job_ecr_access", + "provider_config_key": "aws", + "expressions": { + "policy_arn": { + "constant_value": "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryPowerUser" + }, + "role": { + "references": [ + "aws_iam_role.terra_ci_job" + ] + } + }, + "schema_version": 0 + }, + { + "address": "aws_s3_bucket.terra_ci", + "mode": "managed", + "type": "aws_s3_bucket", + "name": "terra_ci", + "provider_config_key": "aws", + "expressions": { + "acl": { + "constant_value": "private" + }, + "bucket": { + "references": [ + "var.aws_region", + "var.serial_number" + ] + }, + "server_side_encryption_configuration": [ + { + "rule": [ + { + "apply_server_side_encryption_by_default": [ + { + "sse_algorithm": { + "constant_value": "aws:kms" + } + } + ], + "bucket_key_enabled": { + "constant_value": false + } + } + ] + } + ] + }, + "schema_version": 0 + }, + { + "address": "aws_sfn_state_machine.terra_ci_runner", + "mode": "managed", + "type": "aws_sfn_state_machine", + "name": "terra_ci_runner", + "provider_config_key": "aws", + "expressions": { + "definition": { + "references": [ + "aws_codebuild_project.terra_ci", + "aws_codebuild_project.terra_ci" + ] + }, + "name": { + "constant_value": "terra-ci-runner" + }, + "role_arn": { + "references": [ + "aws_iam_role.terra_ci_runner" + ] + } + }, + "schema_version": 0 + }, + { + "address": "data.aws_caller_identity.current", + "mode": "data", + "type": "aws_caller_identity", + "name": "current", + "provider_config_key": "aws", + "schema_version": 0 + }, + { + "address": "data.template_file.terra_ci", + "mode": "data", + "type": "template_file", + "name": "terra_ci", + "provider_config_key": "template", + "expressions": { + "template": {} + }, + "schema_version": 0 + } + ], + "variables": { + "aws_region": {}, + "repo_url": {}, + "serial_number": {} + } + } + } + } diff --git a/tests/test_files/tf_content_generator_files/tfplan-null-example/tf_content.txt b/tests/test_files/tf_content_generator_files/tfplan-null-example/tf_content.txt new file mode 100644 index 00000000..a0ac1510 --- /dev/null +++ b/tests/test_files/tf_content_generator_files/tfplan-null-example/tf_content.txt @@ -0,0 +1,4 @@ +resource "null_resource" "empty" { + triggers = null +} + diff --git a/tests/test_files/tf_content_generator_files/tfplan-null-example/tfplan.json b/tests/test_files/tf_content_generator_files/tfplan-null-example/tfplan.json new file mode 100644 index 00000000..fe05b388 --- /dev/null +++ b/tests/test_files/tf_content_generator_files/tfplan-null-example/tfplan.json @@ -0,0 +1,75 @@ +{ + "format_version": "1.0", + "terraform_version": "1.1.9", + "variables": { + "environment": { + "value": "test" + } + }, + "planned_values": { + "root_module": { + "resources": [ + { + "address": "null_resource.empty", + "mode": "managed", + "type": "null_resource", + "name": "empty", + "provider_name": "registry.terraform.io/hashicorp/null", + "schema_version": 0, + "values": { + "triggers": null + }, + "sensitive_values": {} + } + ] + } + }, + "resource_changes": [ + { + "address": "null_resource.empty", + "mode": "managed", + "type": "null_resource", + "name": "empty", + "provider_name": "registry.terraform.io/hashicorp/null", + "change": { + "actions": [ + "create" + ], + "before": null, + "after": { + "triggers": null + }, + "after_unknown": { + "id": true + }, + "before_sensitive": false, + "after_sensitive": {} + } + } + ], + "configuration": { + "provider_config": { + "aws": { + "name": "aws", + "version_constraint": ">= 4.12.1" + } + }, + "root_module": { + "resources": [ + { + "address": "null_resource.empty", + "mode": "managed", + "type": "null_resource", + "name": "empty", + "provider_config_key": "null", + "schema_version": 0 + } + ], + "variables": { + "environment": { + "description": "The name of the deployment environment" + } + } + } + } +} \ No newline at end of file diff --git a/tests/test_files/tf_content_generator_files/tfplan-update-example/tf_content.txt b/tests/test_files/tf_content_generator_files/tfplan-update-example/tf_content.txt new file mode 100644 index 00000000..0e7a588a --- /dev/null +++ b/tests/test_files/tf_content_generator_files/tfplan-update-example/tf_content.txt @@ -0,0 +1,34 @@ +resource "aws_codebuild_project" "some_projed" { + arn = "arn:aws:codebuild:eu-west-1:719261439472:project/why-my-project-not-working" + artifacts = [{"artifact_identifier": "", "encryption_disabled": true, "location": "terra-ci-artifacts-eu-west-1-000002", "name": "why-my-project-not-working", "namespace_type": "NONE", "override_artifact_name": true, "packaging": "NONE", "path": "", "type": "S3"}] + badge_enabled = false + badge_url = "" + build_timeout = 10 + cache = [{"location": "", "modes": [], "type": "NO_CACHE"}] + description = "Deploy environment configuration" + encryption_key = "arn:aws:kms:eu-west-1:719261439472:alias/aws/s3" + environment = [{"certificate": "", "compute_type": "BUILD_GENERAL1_SMALL", "environment_variable": [], "image": "aws/codebuild/amazonlinux2-x86_64-standard:2.0", "image_pull_credentials_type": "CODEBUILD", "privileged_mode": false, "registry_credential": [], "type": "LINUX_CONTAINER"}] + id = "arn:aws:codebuild:eu-west-1:719261439472:project/why-my-project-not-working" + logs_config = [{"cloudwatch_logs": [{"group_name": "", "status": "ENABLED", "stream_name": ""}], "s3_logs": [{"encryption_disabled": false, "location": "", "status": "DISABLED"}]}] + name = "why-my-project-not-working" + queued_timeout = 480 + secondary_artifacts = [] + secondary_sources = [] + service_role = "arn:aws:iam::719261439472:role/terra_ci_job" + source = [{"auth": [], "buildspec": "version: 0.2\nphases:\n install:\n commands:\n - make install_tools\n build:\n commands:\n - make plan_local resource=$TERRA_CI_RESOURCE\nartifacts:\n files:\n - ./tfplan\n name: $TERRA_CI_BUILD_NAME\n", "git_clone_depth": 1, "git_submodules_config": [], "insecure_ssl": false, "location": "https://github.com/p0tr3c-terraform/terra-ci-single-account.git", "report_build_status": false, "type": "GITHUB"}] + source_version = "" + tags = {} + vpc_config = [] +} + +resource "aws_iam_user" "ci" { + arn = "arn:aws:iam::719261439472:user/ci" + force_destroy = false + id = "ci" + name = "ci" + path = "/" + permissions_boundary = null + tags = {} + unique_id = "AIDA2O52SSXYORYI4EPXD" +} + diff --git a/tests/test_files/tf_content_generator_files/tfplan-update-example/tfplan.json b/tests/test_files/tf_content_generator_files/tfplan-update-example/tfplan.json new file mode 100644 index 00000000..3dde57e2 --- /dev/null +++ b/tests/test_files/tf_content_generator_files/tfplan-update-example/tfplan.json @@ -0,0 +1,736 @@ +{ + "format_version": "0.1", + "terraform_version": "0.15.0", + "variables": { + "AWSServiceRoleForAPIGatewayPresent": { + "value": false + }, + "AWSServiceRoleForAmazonEKSPresent": { + "value": false + }, + "AWSServiceRoleForAutoScalingPresent": { + "value": false + }, + "AWSServiceRoleForOrganizationsPresent": { + "value": false + }, + "AWSServiceRoleForSupportPresent": { + "value": false + }, + "AWSServiceRoleForTrustedAdvisorPresent": { + "value": false + }, + "OrganizationAccountAccessRolePresent": { + "value": false + }, + "aws_account_id": { + "value": "719261439472" + } + }, + "planned_values": { + "root_module": { + "resources": [ + { + "address": "aws_codebuild_project.some_projed", + "mode": "managed", + "type": "aws_codebuild_project", + "name": "some_projed", + "provider_name": "registry.terraform.io/hashicorp/aws", + "schema_version": 0, + "values": { + "arn": "arn:aws:codebuild:eu-west-1:719261439472:project/why-my-project-not-working", + "artifacts": [ + { + "artifact_identifier": "", + "encryption_disabled": true, + "location": "terra-ci-artifacts-eu-west-1-000002", + "name": "why-my-project-not-working", + "namespace_type": "NONE", + "override_artifact_name": true, + "packaging": "NONE", + "path": "", + "type": "S3" + } + ], + "badge_enabled": false, + "badge_url": "", + "build_timeout": 10, + "cache": [ + { + "location": "", + "modes": [], + "type": "NO_CACHE" + } + ], + "description": "Deploy environment configuration", + "encryption_key": "arn:aws:kms:eu-west-1:719261439472:alias/aws/s3", + "environment": [ + { + "certificate": "", + "compute_type": "BUILD_GENERAL1_SMALL", + "environment_variable": [], + "image": "aws/codebuild/amazonlinux2-x86_64-standard:2.0", + "image_pull_credentials_type": "CODEBUILD", + "privileged_mode": false, + "registry_credential": [], + "type": "LINUX_CONTAINER" + } + ], + "id": "arn:aws:codebuild:eu-west-1:719261439472:project/why-my-project-not-working", + "logs_config": [ + { + "cloudwatch_logs": [ + { + "group_name": "", + "status": "ENABLED", + "stream_name": "" + } + ], + "s3_logs": [ + { + "encryption_disabled": false, + "location": "", + "status": "DISABLED" + } + ] + } + ], + "name": "why-my-project-not-working", + "queued_timeout": 480, + "secondary_artifacts": [], + "secondary_sources": [], + "service_role": "arn:aws:iam::719261439472:role/terra_ci_job", + "source": [ + { + "auth": [], + "buildspec": "version: 0.2\nphases:\n install:\n commands:\n - make install_tools\n build:\n commands:\n - make plan_local resource=$TERRA_CI_RESOURCE\nartifacts:\n files:\n - ./tfplan\n name: $TERRA_CI_BUILD_NAME\n", + "git_clone_depth": 1, + "git_submodules_config": [], + "insecure_ssl": false, + "location": "https://github.com/p0tr3c-terraform/terra-ci-single-account.git", + "report_build_status": false, + "type": "GITHUB" + } + ], + "source_version": "", + "tags": {}, + "vpc_config": [] + } + }, + { + "address": "aws_iam_user.ci", + "mode": "managed", + "type": "aws_iam_user", + "name": "ci", + "provider_name": "registry.terraform.io/hashicorp/aws", + "schema_version": 0, + "values": { + "arn": "arn:aws:iam::719261439472:user/ci", + "force_destroy": false, + "id": "ci", + "name": "ci", + "path": "/", + "permissions_boundary": null, + "tags": {}, + "unique_id": "AIDA2O52SSXYORYI4EPXD" + } + } + ] + } + }, + "resource_changes": [ + { + "address": "aws_codebuild_project.some_projed", + "mode": "managed", + "type": "aws_codebuild_project", + "name": "some_projed", + "provider_name": "registry.terraform.io/hashicorp/aws", + "change": { + "actions": [ + "update" + ], + "before": { + "arn": "arn:aws:codebuild:eu-west-1:719261439472:project/why-my-project-not-working", + "artifacts": [ + { + "artifact_identifier": "", + "encryption_disabled": false, + "location": "terra-ci-artifacts-eu-west-1-000002", + "name": "why-my-project-not-working", + "namespace_type": "NONE", + "override_artifact_name": true, + "packaging": "NONE", + "path": "", + "type": "S3" + } + ], + "badge_enabled": false, + "badge_url": "", + "build_timeout": 10, + "cache": [ + { + "location": "", + "modes": [], + "type": "NO_CACHE" + } + ], + "description": "Deploy environment configuration", + "encryption_key": "arn:aws:kms:eu-west-1:719261439472:alias/aws/s3", + "environment": [ + { + "certificate": "", + "compute_type": "BUILD_GENERAL1_SMALL", + "environment_variable": [], + "image": "aws/codebuild/amazonlinux2-x86_64-standard:2.0", + "image_pull_credentials_type": "CODEBUILD", + "privileged_mode": false, + "registry_credential": [], + "type": "LINUX_CONTAINER" + } + ], + "id": "arn:aws:codebuild:eu-west-1:719261439472:project/why-my-project-not-working", + "logs_config": [ + { + "cloudwatch_logs": [ + { + "group_name": "", + "status": "ENABLED", + "stream_name": "" + } + ], + "s3_logs": [ + { + "encryption_disabled": false, + "location": "", + "status": "DISABLED" + } + ] + } + ], + "name": "why-my-project-not-working", + "queued_timeout": 480, + "secondary_artifacts": [], + "secondary_sources": [], + "service_role": "arn:aws:iam::719261439472:role/terra_ci_job", + "source": [ + { + "auth": [], + "buildspec": "version: 0.2\nphases:\n install:\n commands:\n - make install_tools\n build:\n commands:\n - make plan_local resource=$TERRA_CI_RESOURCE\nartifacts:\n files:\n - ./tfplan\n name: $TERRA_CI_BUILD_NAME\n", + "git_clone_depth": 1, + "git_submodules_config": [], + "insecure_ssl": false, + "location": "https://github.com/p0tr3c-terraform/terra-ci-single-account.git", + "report_build_status": false, + "type": "GITHUB" + } + ], + "source_version": "", + "tags": {}, + "vpc_config": [] + }, + "after": { + "arn": "arn:aws:codebuild:eu-west-1:719261439472:project/why-my-project-not-working", + "artifacts": [ + { + "artifact_identifier": "", + "encryption_disabled": true, + "location": "terra-ci-artifacts-eu-west-1-000002", + "name": "why-my-project-not-working", + "namespace_type": "NONE", + "override_artifact_name": true, + "packaging": "NONE", + "path": "", + "type": "S3" + } + ], + "badge_enabled": false, + "badge_url": "", + "build_timeout": 10, + "cache": [ + { + "location": "", + "modes": [], + "type": "NO_CACHE" + } + ], + "description": "Deploy environment configuration", + "encryption_key": "arn:aws:kms:eu-west-1:719261439472:alias/aws/s3", + "environment": [ + { + "certificate": "", + "compute_type": "BUILD_GENERAL1_SMALL", + "environment_variable": [], + "image": "aws/codebuild/amazonlinux2-x86_64-standard:2.0", + "image_pull_credentials_type": "CODEBUILD", + "privileged_mode": false, + "registry_credential": [], + "type": "LINUX_CONTAINER" + } + ], + "id": "arn:aws:codebuild:eu-west-1:719261439472:project/why-my-project-not-working", + "logs_config": [ + { + "cloudwatch_logs": [ + { + "group_name": "", + "status": "ENABLED", + "stream_name": "" + } + ], + "s3_logs": [ + { + "encryption_disabled": false, + "location": "", + "status": "DISABLED" + } + ] + } + ], + "name": "why-my-project-not-working", + "queued_timeout": 480, + "secondary_artifacts": [], + "secondary_sources": [], + "service_role": "arn:aws:iam::719261439472:role/terra_ci_job", + "source": [ + { + "auth": [], + "buildspec": "version: 0.2\nphases:\n install:\n commands:\n - make install_tools\n build:\n commands:\n - make plan_local resource=$TERRA_CI_RESOURCE\nartifacts:\n files:\n - ./tfplan\n name: $TERRA_CI_BUILD_NAME\n", + "git_clone_depth": 1, + "git_submodules_config": [], + "insecure_ssl": false, + "location": "https://github.com/p0tr3c-terraform/terra-ci-single-account.git", + "report_build_status": false, + "type": "GITHUB" + } + ], + "source_version": "", + "tags": {}, + "vpc_config": [] + }, + "after_unknown": {}, + "before_sensitive": { + "artifacts": [ + {} + ], + "cache": [ + { + "modes": [] + } + ], + "environment": [ + { + "environment_variable": [], + "registry_credential": [] + } + ], + "logs_config": [ + { + "cloudwatch_logs": [ + {} + ], + "s3_logs": [ + {} + ] + } + ], + "secondary_artifacts": [], + "secondary_sources": [], + "source": [ + { + "auth": [], + "git_submodules_config": [] + } + ], + "tags": {}, + "vpc_config": [] + }, + "after_sensitive": { + "artifacts": [ + {} + ], + "cache": [ + { + "modes": [] + } + ], + "environment": [ + { + "environment_variable": [], + "registry_credential": [] + } + ], + "logs_config": [ + { + "cloudwatch_logs": [ + {} + ], + "s3_logs": [ + {} + ] + } + ], + "secondary_artifacts": [], + "secondary_sources": [], + "source": [ + { + "auth": [], + "git_submodules_config": [] + } + ], + "tags": {}, + "vpc_config": [] + } + } + }, + { + "address": "aws_iam_user.ci", + "mode": "managed", + "type": "aws_iam_user", + "name": "ci", + "provider_name": "registry.terraform.io/hashicorp/aws", + "change": { + "actions": [ + "no-op" + ], + "before": { + "arn": "arn:aws:iam::719261439472:user/ci", + "force_destroy": false, + "id": "ci", + "name": "ci", + "path": "/", + "permissions_boundary": null, + "tags": {}, + "unique_id": "AIDA2O52SSXYORYI4EPXD" + }, + "after": { + "arn": "arn:aws:iam::719261439472:user/ci", + "force_destroy": false, + "id": "ci", + "name": "ci", + "path": "/", + "permissions_boundary": null, + "tags": {}, + "unique_id": "AIDA2O52SSXYORYI4EPXD" + }, + "after_unknown": {}, + "before_sensitive": { + "tags": {} + }, + "after_sensitive": { + "tags": {} + } + } + } + ], + "prior_state": { + "format_version": "0.1", + "terraform_version": "0.15.0", + "values": { + "root_module": { + "resources": [ + { + "address": "aws_codebuild_project.some_projed", + "mode": "managed", + "type": "aws_codebuild_project", + "name": "some_projed", + "provider_name": "registry.terraform.io/hashicorp/aws", + "schema_version": 0, + "values": { + "arn": "arn:aws:codebuild:eu-west-1:719261439472:project/why-my-project-not-working", + "artifacts": [ + { + "artifact_identifier": "", + "encryption_disabled": false, + "location": "terra-ci-artifacts-eu-west-1-000002", + "name": "why-my-project-not-working", + "namespace_type": "NONE", + "override_artifact_name": true, + "packaging": "NONE", + "path": "", + "type": "S3" + } + ], + "badge_enabled": false, + "badge_url": "", + "build_timeout": 10, + "cache": [ + { + "location": "", + "modes": [], + "type": "NO_CACHE" + } + ], + "description": "Deploy environment configuration", + "encryption_key": "arn:aws:kms:eu-west-1:719261439472:alias/aws/s3", + "environment": [ + { + "certificate": "", + "compute_type": "BUILD_GENERAL1_SMALL", + "environment_variable": [], + "image": "aws/codebuild/amazonlinux2-x86_64-standard:2.0", + "image_pull_credentials_type": "CODEBUILD", + "privileged_mode": false, + "registry_credential": [], + "type": "LINUX_CONTAINER" + } + ], + "id": "arn:aws:codebuild:eu-west-1:719261439472:project/why-my-project-not-working", + "logs_config": [ + { + "cloudwatch_logs": [ + { + "group_name": "", + "status": "ENABLED", + "stream_name": "" + } + ], + "s3_logs": [ + { + "encryption_disabled": false, + "location": "", + "status": "DISABLED" + } + ] + } + ], + "name": "why-my-project-not-working", + "queued_timeout": 480, + "secondary_artifacts": [], + "secondary_sources": [], + "service_role": "arn:aws:iam::719261439472:role/terra_ci_job", + "source": [ + { + "auth": [], + "buildspec": "version: 0.2\nphases:\n install:\n commands:\n - make install_tools\n build:\n commands:\n - make plan_local resource=$TERRA_CI_RESOURCE\nartifacts:\n files:\n - ./tfplan\n name: $TERRA_CI_BUILD_NAME\n", + "git_clone_depth": 1, + "git_submodules_config": [], + "insecure_ssl": false, + "location": "https://github.com/p0tr3c-terraform/terra-ci-single-account.git", + "report_build_status": false, + "type": "GITHUB" + } + ], + "source_version": "", + "tags": {}, + "vpc_config": [] + }, + "depends_on": [ + "data.template_file.terra_ci" + ] + }, + { + "address": "aws_iam_user.ci", + "mode": "managed", + "type": "aws_iam_user", + "name": "ci", + "provider_name": "registry.terraform.io/hashicorp/aws", + "schema_version": 0, + "values": { + "arn": "arn:aws:iam::719261439472:user/ci", + "force_destroy": false, + "id": "ci", + "name": "ci", + "path": "/", + "permissions_boundary": null, + "tags": {}, + "unique_id": "AIDA2O52SSXYORYI4EPXD" + } + }, + { + "address": "data.template_file.terra_ci", + "mode": "data", + "type": "template_file", + "name": "terra_ci", + "provider_name": "registry.terraform.io/hashicorp/template", + "schema_version": 0, + "values": { + "filename": null, + "id": "71c1f84bdc6733e3bcc0e6a7a4cbcdbaa9725d7238cf8b0cf8fe995a826bf534", + "rendered": "version: 0.2\nphases:\n install:\n commands:\n - make install_tools\n build:\n commands:\n - make plan_local resource=$TERRA_CI_RESOURCE\nartifacts:\n files:\n - ./tfplan\n name: $TERRA_CI_BUILD_NAME\n", + "template": "version: 0.2\nphases:\n install:\n commands:\n - make install_tools\n build:\n commands:\n - make plan_local resource=$TERRA_CI_RESOURCE\nartifacts:\n files:\n - ./tfplan\n name: $TERRA_CI_BUILD_NAME\n", + "vars": null + } + } + ] + } + } + }, + "configuration": { + "provider_config": { + "aws": { + "name": "aws", + "expressions": { + "allowed_account_ids": { + "constant_value": [ + "719261439472" + ] + }, + "assume_role": [ + { + "role_arn": { + "constant_value": "arn:aws:iam::719261439472:role/ci" + } + } + ], + "region": { + "constant_value": "eu-west-1" + } + } + } + }, + "root_module": { + "resources": [ + { + "address": "aws_codebuild_project.some_projed", + "mode": "managed", + "type": "aws_codebuild_project", + "name": "some_projed", + "provider_config_key": "aws", + "expressions": { + "artifacts": [ + { + "encryption_disabled": { + "constant_value": true + }, + "location": { + "constant_value": "terra-ci-artifacts-eu-west-1-000002" + }, + "override_artifact_name": { + "constant_value": true + }, + "type": { + "constant_value": "S3" + } + } + ], + "build_timeout": { + "constant_value": "10" + }, + "description": { + "constant_value": "Deploy environment configuration" + }, + "environment": [ + { + "compute_type": { + "constant_value": "BUILD_GENERAL1_SMALL" + }, + "image": { + "constant_value": "aws/codebuild/amazonlinux2-x86_64-standard:2.0" + }, + "image_pull_credentials_type": { + "constant_value": "CODEBUILD" + }, + "privileged_mode": { + "constant_value": false + }, + "type": { + "constant_value": "LINUX_CONTAINER" + } + } + ], + "logs_config": [ + { + "cloudwatch_logs": [ + { + "status": { + "constant_value": "ENABLED" + } + } + ], + "s3_logs": [ + { + "encryption_disabled": { + "constant_value": false + }, + "status": { + "constant_value": "DISABLED" + } + } + ] + } + ], + "name": { + "constant_value": "why-my-project-not-working" + }, + "service_role": { + "constant_value": "arn:aws:iam::719261439472:role/terra_ci_job" + }, + "source": [ + { + "buildspec": { + "references": [ + "data.template_file.terra_ci" + ] + }, + "git_clone_depth": { + "constant_value": 1 + }, + "insecure_ssl": { + "constant_value": false + }, + "location": { + "constant_value": "https://github.com/p0tr3c-terraform/terra-ci-single-account.git" + }, + "report_build_status": { + "constant_value": false + }, + "type": { + "constant_value": "GITHUB" + } + } + ] + }, + "schema_version": 0 + }, + { + "address": "aws_iam_user.ci", + "mode": "managed", + "type": "aws_iam_user", + "name": "ci", + "provider_config_key": "aws", + "expressions": { + "name": { + "constant_value": "ci" + } + }, + "schema_version": 0 + }, + { + "address": "data.template_file.terra_ci", + "mode": "data", + "type": "template_file", + "name": "terra_ci", + "provider_config_key": "template", + "expressions": { + "template": {} + }, + "schema_version": 0 + } + ], + "variables": { + "AWSServiceRoleForAPIGatewayPresent": { + "default": false + }, + "AWSServiceRoleForAmazonEKSPresent": { + "default": false + }, + "AWSServiceRoleForAutoScalingPresent": { + "default": false + }, + "AWSServiceRoleForOrganizationsPresent": { + "default": false + }, + "AWSServiceRoleForSupportPresent": { + "default": false + }, + "AWSServiceRoleForTrustedAdvisorPresent": { + "default": false + }, + "OrganizationAccountAccessRolePresent": { + "default": false + }, + "aws_account_id": {} + } + } + } +} \ No newline at end of file diff --git a/tests/test_files/zip_content/__init__.py b/tests/test_files/zip_content/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_files/zip_content/sast.py b/tests/test_files/zip_content/sast.py index aa6df96e..2de8a5ca 100644 --- a/tests/test_files/zip_content/sast.py +++ b/tests/test_files/zip_content/sast.py @@ -1,4 +1,3 @@ import requests - -requests.get('https://slack.com/api/conversations.list', verify=False) +requests.get('https://slack.com/api/conversations.list', verify=False) # noqa: S113, S501 diff --git a/tests/test_files/zip_content/secrets.py b/tests/test_files/zip_content/secrets.py index a8c7d82a..627ee470 100644 --- a/tests/test_files/zip_content/secrets.py +++ b/tests/test_files/zip_content/secrets.py @@ -1 +1 @@ -slack_bot_token = "xoxb-1234518014707-1234518014707-M14123412341234Fra15WS" +slack_bot_token = 'xoxb-1234518014707-1234518014707-M14123412341234Fra15WS' diff --git a/tests/test_models.py b/tests/test_models.py index 270b9780..38998250 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -1,21 +1,21 @@ -from cycode.cyclient.models import ResourcesCollection, InternalMetadata, K8SResource +from cycode.cyclient.models import InternalMetadata, K8SResource, ResourcesCollection from tests import PODS_MOCK -def test_batch_resources_to_json(): +def test_batch_resources_to_json() -> None: batch = ResourcesCollection('pod', 'default', PODS_MOCK, 77777) json_dict = batch.to_json() - assert 'resources' in json_dict.keys() - assert 'namespace' in json_dict.keys() - assert 'total_count' in json_dict.keys() - assert 'type' in json_dict.keys() + assert 'resources' in json_dict + assert 'namespace' in json_dict + assert 'total_count' in json_dict + assert 'type' in json_dict assert json_dict['total_count'] == 77777 assert json_dict['type'] == 'pod' assert json_dict['namespace'] == 'default' assert json_dict['resources'][0]['name'] == 'pod_name_1' -def test_internal_metadata_to_json(): +def test_internal_metadata_to_json() -> None: resource = K8SResource('nginx-template-123-456', 'pod', 'cycode', {}) resource.internal_metadata = InternalMetadata('nginx-template', 'deployment') batch = ResourcesCollection('pod', 'cycode', [resource], 1) diff --git a/tests/test_models_deserialization.py b/tests/test_models_deserialization.py new file mode 100644 index 00000000..4c7dcd72 --- /dev/null +++ b/tests/test_models_deserialization.py @@ -0,0 +1,451 @@ +from cycode.cyclient.models import ( + ApiToken, + ApiTokenGenerationPollingResponse, + ApiTokenGenerationPollingResponseSchema, + ApiTokenSchema, + AuthenticationSession, + AuthenticationSessionSchema, + ClassificationData, + ClassificationDataSchema, + Detection, + DetectionRule, + DetectionRuleSchema, + DetectionSchema, + Member, + MemberDetails, + MemberSchema, + ReportExecution, + ReportExecutionSchema, + RequestedMemberDetailsResultSchema, + RequestedSbomReportResultSchema, + SbomReport, + SbomReportStorageDetails, + SbomReportStorageDetailsSchema, + ScanConfiguration, + ScanConfigurationSchema, + ScanInitializationResponse, + ScanInitializationResponseSchema, + ScanResult, + ScanResultSchema, + ScanResultsSyncFlow, + ScanResultsSyncFlowSchema, + SupportedModulesPreferences, + SupportedModulesPreferencesSchema, + UserAgentOption, + UserAgentOptionScheme, +) + +# --- DetectionSchema --- + + +def test_detection_schema_load() -> None: + raw = { + 'id': 'det-123', + 'message': 'API key exposed', + 'type': 'secret', + 'severity': 'critical', + 'detection_type_id': 'secret-1', + 'detection_details': {'alert': True, 'value': 'sk_live_xxx'}, + 'detection_rule_id': 'rule-456', + } + result = DetectionSchema().load(raw) + assert isinstance(result, Detection) + assert result.id == 'det-123' + assert result.message == 'API key exposed' + assert result.type == 'secret' + assert result.severity == 'critical' + assert result.detection_type_id == 'secret-1' + assert result.detection_details == {'alert': True, 'value': 'sk_live_xxx'} + assert result.detection_rule_id == 'rule-456' + + +def test_detection_schema_load_defaults() -> None: + raw = { + 'message': 'Vulnerability found', + 'type': 'sca', + 'detection_type_id': 'vuln-1', + 'detection_details': {}, + 'detection_rule_id': 'rule-789', + } + result = DetectionSchema().load(raw) + assert result.id is None + assert result.severity is None + + +def test_detection_schema_excludes_unknown_fields() -> None: + raw = { + 'message': 'Test', + 'type': 'test', + 'detection_type_id': 'test-1', + 'detection_details': {}, + 'detection_rule_id': 'test-rule', + 'unknown_field': 'should_be_ignored', + 'another_unknown': 123, + } + result = DetectionSchema().load(raw) + assert isinstance(result, Detection) + assert not hasattr(result, 'unknown_field') + + +def test_detection_has_alert_true() -> None: + detection = Detection( + detection_type_id='secret-1', + type='secret', + message='Key found', + detection_details={'alert': {'severity': 'high'}}, + detection_rule_id='rule-1', + ) + assert detection.has_alert is True + + +def test_detection_has_alert_false() -> None: + detection = Detection( + detection_type_id='license-1', + type='sca', + message='License issue', + detection_details={'license': 'GPL'}, + detection_rule_id='rule-2', + ) + assert detection.has_alert is False + + +def test_detection_repr() -> None: + detection = Detection( + detection_type_id='secret-1', + type='secret', + message='API key exposed', + detection_details={'value': 'sk_live_xxx'}, + detection_rule_id='rule-1', + severity='critical', + ) + repr_str = repr(detection) + assert 'secret' in repr_str + assert 'critical' in repr_str + assert 'API key exposed' in repr_str + assert 'rule-1' in repr_str + + +# --- ScanResultSchema --- + + +def test_scan_result_schema_load_with_detections() -> None: + raw = { + 'did_detect': True, + 'scan_id': 'scan-abc', + 'detections': [ + { + 'id': 'det-1', + 'message': 'Secret found', + 'type': 'secret', + 'detection_type_id': 'secret-1', + 'detection_details': {'alert': {}}, + 'detection_rule_id': 'rule-1', + } + ], + 'err': '', + } + result = ScanResultSchema().load(raw) + assert isinstance(result, ScanResult) + assert result.did_detect is True + assert result.scan_id == 'scan-abc' + assert len(result.detections) == 1 + assert isinstance(result.detections[0], Detection) + assert result.detections[0].id == 'det-1' + + +def test_scan_result_schema_load_no_detections() -> None: + raw = { + 'did_detect': False, + 'scan_id': 'scan-def', + 'detections': None, + 'err': 'No files to scan', + } + result = ScanResultSchema().load(raw) + assert result.did_detect is False + assert result.detections is None + assert result.err == 'No files to scan' + + +def test_scan_result_schema_excludes_unknown_fields() -> None: + raw = { + 'did_detect': False, + 'scan_id': 'scan-1', + 'detections': None, + 'err': '', + 'extra_field': 'ignored', + } + result = ScanResultSchema().load(raw) + assert isinstance(result, ScanResult) + + +# --- ScanInitializationResponseSchema --- + + +def test_scan_initialization_response_schema_load() -> None: + raw = {'scan_id': 'scan-init-123', 'err': ''} + result = ScanInitializationResponseSchema().load(raw) + assert isinstance(result, ScanInitializationResponse) + assert result.scan_id == 'scan-init-123' + + +# --- AuthenticationSessionSchema --- + + +def test_authentication_session_schema_load() -> None: + raw = {'session_id': 'sess-123'} + result = AuthenticationSessionSchema().load(raw) + assert isinstance(result, AuthenticationSession) + assert result.session_id == 'sess-123' + + +# --- ApiTokenSchema (tests data_key mapping) --- + + +def test_api_token_schema_load_data_key() -> None: + raw = { + 'clientId': 'client-123', + 'secret': 'secret-456', + 'description': 'My API Token', + } + result = ApiTokenSchema().load(raw) + assert isinstance(result, ApiToken) + assert result.client_id == 'client-123' + assert result.secret == 'secret-456' + assert result.description == 'My API Token' + + +# --- ApiTokenGenerationPollingResponseSchema (nested) --- + + +def test_api_token_generation_polling_schema_load() -> None: + raw = { + 'status': 'completed', + 'api_token': { + 'clientId': 'client-abc', + 'secret': 'secret-xyz', + 'description': 'Generated token', + }, + } + result = ApiTokenGenerationPollingResponseSchema().load(raw) + assert isinstance(result, ApiTokenGenerationPollingResponse) + assert result.status == 'completed' + assert isinstance(result.api_token, ApiToken) + assert result.api_token.client_id == 'client-abc' + + +def test_api_token_generation_polling_schema_load_null_token() -> None: + raw = { + 'status': 'pending', + 'api_token': None, + } + result = ApiTokenGenerationPollingResponseSchema().load(raw) + assert result.status == 'pending' + assert result.api_token is None + + +# --- SbomReportStorageDetailsSchema / ReportExecutionSchema / RequestedSbomReportResultSchema --- + + +def test_sbom_report_storage_details_schema_load() -> None: + raw = {'path': '/reports/sbom.json', 'folder': '/reports', 'size': 4096} + result = SbomReportStorageDetailsSchema().load(raw) + assert isinstance(result, SbomReportStorageDetails) + assert result.path == '/reports/sbom.json' + assert result.size == 4096 + + +def test_report_execution_schema_load() -> None: + raw = { + 'id': 1, + 'status': 'completed', + 'error_message': None, + 'status_message': 'Success', + 'storage_details': {'path': '/reports/sbom.json', 'folder': '/reports', 'size': 4096}, + } + result = ReportExecutionSchema().load(raw) + assert isinstance(result, ReportExecution) + assert result.id == 1 + assert result.status == 'completed' + assert isinstance(result.storage_details, SbomReportStorageDetails) + + +def test_requested_sbom_report_result_schema_load() -> None: + raw = { + 'report_executions': [ + { + 'id': 1, + 'status': 'completed', + 'error_message': None, + 'status_message': 'Done', + 'storage_details': {'path': '/r/sbom.json', 'folder': '/r', 'size': 1024}, + }, + { + 'id': 2, + 'status': 'failed', + 'error_message': 'Timeout', + 'status_message': None, + 'storage_details': None, + }, + ] + } + result = RequestedSbomReportResultSchema().load(raw) + assert isinstance(result, SbomReport) + assert len(result.report_executions) == 2 + assert result.report_executions[0].storage_details.path == '/r/sbom.json' + assert result.report_executions[1].error_message == 'Timeout' + assert result.report_executions[1].storage_details is None + + +# --- UserAgentOptionScheme --- + + +def test_user_agent_option_schema_load() -> None: + raw = { + 'app_name': 'vscode_extension', + 'app_version': '0.2.3', + 'env_name': 'Visual Studio Code', + 'env_version': '1.78.2', + } + result = UserAgentOptionScheme().load(raw) + assert isinstance(result, UserAgentOption) + assert result.app_name == 'vscode_extension' + assert 'vscode_extension' in result.user_agent_suffix + assert 'AppVersion: 0.2.3' in result.user_agent_suffix + + +# --- MemberSchema / RequestedMemberDetailsResultSchema --- + + +def test_member_schema_load() -> None: + raw = {'external_id': 'user-ext-123'} + result = MemberSchema().load(raw) + assert isinstance(result, Member) + assert result.external_id == 'user-ext-123' + + +def test_requested_member_details_schema_load() -> None: + raw = { + 'items': [{'external_id': 'u1'}, {'external_id': 'u2'}], + 'page_size': 50, + 'next_page_token': 'token-abc', + } + result = RequestedMemberDetailsResultSchema().load(raw) + assert isinstance(result, MemberDetails) + assert len(result.items) == 2 + assert result.page_size == 50 + assert result.next_page_token == 'token-abc' + + +def test_requested_member_details_schema_load_null_token() -> None: + raw = { + 'items': [], + 'page_size': 50, + 'next_page_token': None, + } + result = RequestedMemberDetailsResultSchema().load(raw) + assert result.next_page_token is None + + +# --- ClassificationDataSchema / DetectionRuleSchema --- + + +def test_classification_data_schema_load() -> None: + raw = {'severity': 'high'} + result = ClassificationDataSchema().load(raw) + assert isinstance(result, ClassificationData) + assert result.severity == 'high' + + +def test_detection_rule_schema_load() -> None: + raw = { + 'classification_data': [{'severity': 'high'}, {'severity': 'medium'}], + 'detection_rule_id': 'rule-123', + 'custom_remediation_guidelines': 'Rotate the key', + 'remediation_guidelines': 'See docs', + 'description': 'Exposed API key', + 'policy_name': 'secrets-policy', + 'display_name': 'API Key Exposure', + } + result = DetectionRuleSchema().load(raw) + assert isinstance(result, DetectionRule) + assert len(result.classification_data) == 2 + assert result.classification_data[0].severity == 'high' + assert result.detection_rule_id == 'rule-123' + assert result.custom_remediation_guidelines == 'Rotate the key' + + +def test_detection_rule_schema_load_optional_nulls() -> None: + raw = { + 'classification_data': [{'severity': 'low'}], + 'detection_rule_id': 'rule-456', + 'custom_remediation_guidelines': None, + 'remediation_guidelines': None, + 'description': None, + 'policy_name': None, + 'display_name': None, + } + result = DetectionRuleSchema().load(raw) + assert result.custom_remediation_guidelines is None + assert result.display_name is None + + +# --- ScanResultsSyncFlowSchema --- + + +def test_scan_results_sync_flow_schema_load() -> None: + raw = { + 'id': 'sync-123', + 'detection_messages': [{'msg': 'found secret'}, {'msg': 'found vuln'}], + } + result = ScanResultsSyncFlowSchema().load(raw) + assert isinstance(result, ScanResultsSyncFlow) + assert result.id == 'sync-123' + assert len(result.detection_messages) == 2 + + +# --- SupportedModulesPreferencesSchema --- + + +def test_supported_modules_preferences_schema_load() -> None: + raw = { + 'secret_scanning': True, + 'leak_scanning': True, + 'iac_scanning': False, + 'sca_scanning': True, + 'ci_cd_scanning': False, + 'sast_scanning': True, + 'container_scanning': False, + 'access_review': True, + 'asoc': False, + 'cimon': True, + 'ai_machine_learning': True, + 'ai_large_language_model': False, + } + result = SupportedModulesPreferencesSchema().load(raw) + assert isinstance(result, SupportedModulesPreferences) + assert result.secret_scanning is True + assert result.iac_scanning is False + assert result.ai_large_language_model is False + + +# --- ScanConfigurationSchema --- + + +def test_scan_configuration_schema_load() -> None: + raw = { + 'scannable_extensions': ['.py', '.js', '.ts'], + 'is_cycode_ignore_allowed': True, + } + result = ScanConfigurationSchema().load(raw) + assert isinstance(result, ScanConfiguration) + assert result.scannable_extensions == ['.py', '.js', '.ts'] + assert result.is_cycode_ignore_allowed is True + + +def test_scan_configuration_schema_load_defaults() -> None: + raw = { + 'scannable_extensions': None, + } + result = ScanConfigurationSchema().load(raw) + assert result.scannable_extensions is None + assert result.is_cycode_ignore_allowed is True # load_default=True diff --git a/tests/test_performance_get_all_files.py b/tests/test_performance_get_all_files.py new file mode 100644 index 00000000..b0e8653d --- /dev/null +++ b/tests/test_performance_get_all_files.py @@ -0,0 +1,94 @@ +import glob +import logging +import os +import timeit +from pathlib import Path +from typing import Union + +logger = logging.getLogger(__name__) +logging.basicConfig(level=logging.INFO) + + +def filter_files(paths: list[Union[Path, str]]) -> list[str]: + return [str(path) for path in paths if os.path.isfile(path)] + + +def get_all_files_glob(path: Union[Path, str]) -> list[str]: + # DOESN'T RETURN HIDDEN FILES. CAN'T BE USED + # and doesn't show the best performance + if not str(path).endswith(os.sep): + path = f'{path}{os.sep}' + + return filter_files(glob.glob(f'{path}**', recursive=True)) + + +def get_all_files_walk(path: str) -> list[str]: + files = [] + + for root, _, filenames in os.walk(path): + for filename in filenames: + files.append(os.path.join(root, filename)) + + return files + + +def get_all_files_listdir(path: str) -> list[str]: + files = [] + + def _(sub_path: str) -> None: + items = os.listdir(sub_path) + + for item in items: + item_path = os.path.join(sub_path, item) + + if os.path.isfile(item_path): + files.append(item_path) + elif os.path.isdir(item_path): + _(item_path) + + _(path) + return files + + +def get_all_files_rglob(path: str) -> list[str]: + return filter_files(list(Path(path).rglob(r'*'))) + + +def test_get_all_files_performance(test_files_path: str) -> None: + results: dict[str, tuple[int, float]] = {} + for func in { + get_all_files_rglob, + get_all_files_listdir, + get_all_files_walk, + }: + name = func.__name__ + start_time = timeit.default_timer() + + files_count = len(func(test_files_path)) + + executed_time = timeit.default_timer() - start_time + results[name] = (files_count, executed_time) + + logger.info('Time result %s: %s', name, executed_time) + logger.info('Files count %s: %s', name, files_count) + + files_counts = [result[0] for result in results.values()] + assert len(set(files_counts)) == 1 # all should be equal + + logger.info('Benchmark TOP with (%s) files:', files_counts[0]) + for func_name, result in sorted(results.items(), key=lambda x: x[1][1]): + logger.info('- %s: %s', func_name, result[1]) + + # according to my (MarshalX) local tests, the fastest is get_all_files_walk + + +if __name__ == '__main__': + # provide a path with thousands of files + huge_dir_path = '/Users/ilyasiamionau/projects/cycode/' + test_get_all_files_performance(huge_dir_path) + + # Output: + # INFO:__main__:Benchmark TOP with (94882) files: + # INFO:__main__:- get_all_files_walk: 0.717258458 + # INFO:__main__:- get_all_files_listdir: 1.4648628330000002 + # INFO:__main__:- get_all_files_rglob: 2.368291458 diff --git a/tests/test_zip_file.py b/tests/test_zip_file.py index c05089f5..15c53c17 100644 --- a/tests/test_zip_file.py +++ b/tests/test_zip_file.py @@ -1,22 +1,22 @@ import os -from cycode.cli import zip_file +from cycode.cli.utils.path_utils import concat_unique_id -def test_concat_unique_id_to_file_with_leading_slash(): +def test_concat_unique_id_to_file_with_leading_slash() -> None: filename = os.path.join('path', 'to', 'file') # we should care about slash characters in tests unique_id = 'unique_id' expected_path = os.path.join(unique_id, filename) filename = os.sep + filename - assert zip_file.concat_unique_id(filename, unique_id) == expected_path + assert concat_unique_id(filename, unique_id) == expected_path -def test_concat_unique_id_to_file_without_leading_slash(): +def test_concat_unique_id_to_file_without_leading_slash() -> None: filename = os.path.join('path', 'to', 'file') # we should care about slash characters in tests unique_id = 'unique_id' expected_path = os.path.join(unique_id, *filename.split('/')) - assert zip_file.concat_unique_id(filename, unique_id) == expected_path + assert concat_unique_id(filename, unique_id) == expected_path diff --git a/tests/user_settings/test_configuration_manager.py b/tests/user_settings/test_configuration_manager.py index 18c9cb9f..5aa7f6a8 100644 --- a/tests/user_settings/test_configuration_manager.py +++ b/tests/user_settings/test_configuration_manager.py @@ -1,7 +1,11 @@ -from mock import Mock +from typing import TYPE_CHECKING, Optional +from unittest.mock import Mock -from cycode.cli.user_settings.configuration_manager import ConfigurationManager from cycode.cli.consts import DEFAULT_CYCODE_API_URL +from cycode.cli.user_settings.configuration_manager import ConfigurationManager + +if TYPE_CHECKING: + from pytest_mock import MockerFixture """ we check for base url in the three places, in the following order: @@ -14,10 +18,11 @@ GLOBAL_CONFIG_BASE_URL_VALUE = 'url_from_global_config_file' -def test_get_base_url_from_environment_variable(mocker): +def test_get_base_url_from_environment_variable(mocker: 'MockerFixture') -> None: # Arrange - configuration_manager = _configure_mocks(mocker, ENV_VARS_BASE_URL_VALUE, LOCAL_CONFIG_FILE_BASE_URL_VALUE, - GLOBAL_CONFIG_BASE_URL_VALUE) + configuration_manager = _configure_mocks( + mocker, ENV_VARS_BASE_URL_VALUE, LOCAL_CONFIG_FILE_BASE_URL_VALUE, GLOBAL_CONFIG_BASE_URL_VALUE + ) # Act result = configuration_manager.get_cycode_api_url() @@ -26,10 +31,11 @@ def test_get_base_url_from_environment_variable(mocker): assert result == ENV_VARS_BASE_URL_VALUE -def test_get_base_url_from_local_config(mocker): +def test_get_base_url_from_local_config(mocker: 'MockerFixture') -> None: # Arrange - configuration_manager = _configure_mocks(mocker, None, LOCAL_CONFIG_FILE_BASE_URL_VALUE, - GLOBAL_CONFIG_BASE_URL_VALUE) + configuration_manager = _configure_mocks( + mocker, None, LOCAL_CONFIG_FILE_BASE_URL_VALUE, GLOBAL_CONFIG_BASE_URL_VALUE + ) # Act result = configuration_manager.get_cycode_api_url() @@ -38,7 +44,7 @@ def test_get_base_url_from_local_config(mocker): assert result == LOCAL_CONFIG_FILE_BASE_URL_VALUE -def test_get_base_url_from_global_config(mocker): +def test_get_base_url_from_global_config(mocker: 'MockerFixture') -> None: # Arrange configuration_manager = _configure_mocks(mocker, None, None, GLOBAL_CONFIG_BASE_URL_VALUE) @@ -49,7 +55,7 @@ def test_get_base_url_from_global_config(mocker): assert result == GLOBAL_CONFIG_BASE_URL_VALUE -def test_get_base_url_not_configured(mocker): +def test_get_base_url_not_configured(mocker: 'MockerFixture') -> None: # Arrange configuration_manager = _configure_mocks(mocker, None, None, None) @@ -60,12 +66,15 @@ def test_get_base_url_not_configured(mocker): assert result == DEFAULT_CYCODE_API_URL -def _configure_mocks(mocker, - expected_env_var_base_url, - expected_local_config_file_base_url, - expected_global_config_file_base_url): - mocker.patch.object(ConfigurationManager, 'get_api_url_from_environment_variables', - return_value=expected_env_var_base_url) +def _configure_mocks( + mocker: 'MockerFixture', + expected_env_var_base_url: Optional[str], + expected_local_config_file_base_url: Optional[str], + expected_global_config_file_base_url: Optional[str], +) -> ConfigurationManager: + mocker.patch.object( + ConfigurationManager, 'get_api_url_from_environment_variables', return_value=expected_env_var_base_url + ) configuration_manager = ConfigurationManager() configuration_manager.local_config_file_manager = Mock() configuration_manager.local_config_file_manager.get_api_url.return_value = expected_local_config_file_base_url diff --git a/tests/user_settings/test_user_settings_commands.py b/tests/user_settings/test_user_settings_commands.py deleted file mode 100644 index 600284d5..00000000 --- a/tests/user_settings/test_user_settings_commands.py +++ /dev/null @@ -1,104 +0,0 @@ -from click.testing import CliRunner - -from cycode.cli.user_settings.user_settings_commands import set_credentials - - -def test_set_credentials_no_exist_credentials_in_file(mocker): - # Arrange - client_id_user_input = "new client id" - client_secret_user_input = "new client secret" - mocker.patch('cycode.cli.user_settings.credentials_manager.CredentialsManager.get_credentials_from_file', - return_value=(None, None)) - - # side effect - multiple return values, each item in the list represent return of a call - mocker.patch('click.prompt', side_effect=[client_id_user_input, client_secret_user_input]) - mocked_update_credentials = mocker.patch( - 'cycode.cli.user_settings.credentials_manager.CredentialsManager.update_credentials_file') - click = CliRunner() - - # Act - click.invoke(set_credentials) - - # Assert - mocked_update_credentials.assert_called_once_with(client_id_user_input, client_secret_user_input) - - -def test_set_credentials_update_current_credentials_in_file(mocker): - # Arrange - client_id_user_input = "new client id" - client_secret_user_input = "new client secret" - mocker.patch('cycode.cli.user_settings.credentials_manager.CredentialsManager.get_credentials_from_file', - return_value=('client id file', 'client secret file')) - - # side effect - multiple return values, each item in the list represent return of a call - mocker.patch('click.prompt', side_effect=[client_id_user_input, client_secret_user_input]) - mocked_update_credentials = mocker.patch( - 'cycode.cli.user_settings.credentials_manager.CredentialsManager.update_credentials_file') - click = CliRunner() - - # Act - click.invoke(set_credentials) - - # Assert - mocked_update_credentials.assert_called_once_with(client_id_user_input, client_secret_user_input) - - -def test_set_credentials_update_only_client_id(mocker): - # Arrange - client_id_user_input = "new client id" - current_client_id = 'client secret file' - mocker.patch('cycode.cli.user_settings.credentials_manager.CredentialsManager.get_credentials_from_file', - return_value=('client id file', 'client secret file')) - - # side effect - multiple return values, each item in the list represent return of a call - mocker.patch('click.prompt', side_effect=[client_id_user_input, '']) - mocked_update_credentials = mocker.patch( - 'cycode.cli.user_settings.credentials_manager.CredentialsManager.update_credentials_file') - click = CliRunner() - - # Act - click.invoke(set_credentials) - - # Assert - mocked_update_credentials.assert_called_once_with(client_id_user_input, current_client_id) - - -def test_set_credentials_update_only_client_secret(mocker): - # Arrange - client_secret_user_input = "new client secret" - current_client_id = 'client secret file' - mocker.patch('cycode.cli.user_settings.credentials_manager.CredentialsManager.get_credentials_from_file', - return_value=(current_client_id, 'client secret file')) - - # side effect - multiple return values, each item in the list represent return of a call - mocker.patch('click.prompt', side_effect=['', client_secret_user_input]) - mocked_update_credentials = mocker.patch( - 'cycode.cli.user_settings.credentials_manager.CredentialsManager.update_credentials_file') - click = CliRunner() - - # Act - click.invoke(set_credentials) - - # Assert - mocked_update_credentials.assert_called_once_with(current_client_id, client_secret_user_input) - - -def test_set_credentials_should_not_update_file(mocker): - # Arrange - client_id_user_input = "" - client_secret_user_input = "" - mocker.patch('cycode.cli.user_settings.credentials_manager.CredentialsManager.get_credentials_from_file', - return_value=('client id file', 'client secret file')) - - # side effect - multiple return values, each item in the list represent return of a call - mocker.patch('click.prompt', side_effect=[client_id_user_input, client_secret_user_input]) - mocked_update_credentials = mocker.patch( - 'cycode.cli.user_settings.credentials_manager.CredentialsManager.update_credentials_file') - click = CliRunner() - - # Act - click.invoke(set_credentials) - - # Assert - assert not mocked_update_credentials.called - diff --git a/tests/utils/test_binary_utils.py b/tests/utils/test_binary_utils.py new file mode 100644 index 00000000..c8fa7e53 --- /dev/null +++ b/tests/utils/test_binary_utils.py @@ -0,0 +1,42 @@ +import pytest + +from cycode.cli.utils.binary_utils import is_binary_string + + +@pytest.mark.parametrize( + ('data', 'expected'), + [ + # Empty / None-ish + (b'', False), + (None, False), + # Plain ASCII text + (b'Hello, world!', False), + (b'print("hello")\nfor i in range(10):\n pass\n', False), + # Whitespace-heavy text (tabs, newlines) is not binary + (b'\t\t\n\n\r\n some text\n', False), + # UTF-8 multibyte text (accented, CJK, emoji) + ('café résumé naïve'.encode(), False), + ('日本語テキスト'.encode(), False), + ('🎉🚀💻'.encode(), False), + # BOM-marked UTF-16/32 text is not binary + ('\ufeffHello UTF-16'.encode('utf-16-le'), False), + ('\ufeffHello UTF-16'.encode('utf-16-be'), False), + ('\ufeffHello UTF-32'.encode('utf-32-le'), False), + ('\ufeffHello UTF-32'.encode('utf-32-be'), False), + # Null bytes → binary + (b'\x00', True), + (b'hello\x00world', True), + (b'\x00\x01\x02\x03', True), + # 0xff in otherwise normal data → binary + (b'hello\xffworld', True), + # Mostly control chars + invalid UTF-8 → binary + (b'\x01\x02\x03\x04\x05\x06\x07\x0e\x0f\x10' * 10 + b'\x80', True), + # Real binary format headers + (b'\x89PNG\r\n\x1a\n' + b'\x00' * 100, True), + (b'\x7fELF' + b'\x00' * 100, True), + # DS_Store-like: null-byte-heavy valid UTF-8 → still binary + (b'\x00\x00\x00\x01Bud1' + b'\x00' * 100, True), + ], +) +def test_is_binary_string(data: bytes, expected: bool) -> None: + assert is_binary_string(data) is expected diff --git a/tests/utils/test_git_proxy.py b/tests/utils/test_git_proxy.py new file mode 100644 index 00000000..62416361 --- /dev/null +++ b/tests/utils/test_git_proxy.py @@ -0,0 +1,53 @@ +import os +import tempfile + +import git as real_git +import pytest + +from cycode.cli.utils.git_proxy import _GIT_ERROR_MESSAGE, GitProxyError, _DummyGitProxy, _GitProxy, get_git_proxy + + +def test_get_git_proxy() -> None: + proxy = get_git_proxy(git_module=None) + assert isinstance(proxy, _DummyGitProxy) + + proxy2 = get_git_proxy(git_module=real_git) + assert isinstance(proxy2, _GitProxy) + + +def test_dummy_git_proxy() -> None: + proxy = _DummyGitProxy() + + with pytest.raises(RuntimeError) as exc: + proxy.get_repo() + assert str(exc.value) == _GIT_ERROR_MESSAGE + + with pytest.raises(RuntimeError) as exc2: + proxy.get_null_tree() + assert str(exc2.value) == _GIT_ERROR_MESSAGE + + assert proxy.get_git_command_error() is GitProxyError + assert proxy.get_invalid_git_repository_error() is GitProxyError + + +def test_git_proxy() -> None: + proxy = _GitProxy() + + repo = proxy.get_repo(os.getcwd(), search_parent_directories=True) + assert isinstance(repo, real_git.Repo) + + assert proxy.get_null_tree() is real_git.NULL_TREE + + assert proxy.get_git_command_error() is real_git.GitCommandError + assert proxy.get_invalid_git_repository_error() is real_git.InvalidGitRepositoryError + + with tempfile.TemporaryDirectory() as tmpdir: + with pytest.raises(real_git.InvalidGitRepositoryError): + proxy.get_repo(tmpdir) + with pytest.raises(proxy.get_invalid_git_repository_error()): + proxy.get_repo(tmpdir) + + with pytest.raises(real_git.GitCommandError): + repo.git.show('blabla') + with pytest.raises(proxy.get_git_command_error()): + repo.git.show('blabla') diff --git a/tests/utils/test_ignore_utils.py b/tests/utils/test_ignore_utils.py new file mode 100644 index 00000000..6988e1aa --- /dev/null +++ b/tests/utils/test_ignore_utils.py @@ -0,0 +1,176 @@ +# Copyright (C) 2017 Jelmer Vernooij +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Modified (rewritten to pytest + pyfakefs) from https://github.com/jelmer/dulwich/blob/master/tests/test_ignore.py + +import os +import re +from io import BytesIO +from typing import TYPE_CHECKING + +import pytest + +from cycode.cli.utils.ignore_utils import ( + IgnoreFilter, + IgnoreFilterManager, + Pattern, + match_pattern, + read_ignore_patterns, + translate, +) + +if TYPE_CHECKING: + from pyfakefs.fake_filesystem import FakeFilesystem + +POSITIVE_MATCH_TESTS = [ + (b'foo.c', b'*.c'), + (b'.c', b'*.c'), + (b'foo/foo.c', b'*.c'), + (b'foo/foo.c', b'foo.c'), + (b'foo.c', b'/*.c'), + (b'foo.c', b'/foo.c'), + (b'foo.c', b'foo.c'), + (b'foo.c', b'foo.[ch]'), + (b'foo/bar/bla.c', b'foo/**'), + (b'foo/bar/bla/blie.c', b'foo/**/blie.c'), + (b'foo/bar/bla.c', b'**/bla.c'), + (b'bla.c', b'**/bla.c'), + (b'foo/bar', b'foo/**/bar'), + (b'foo/bla/bar', b'foo/**/bar'), + (b'foo/bar/', b'bar/'), + (b'foo/bar/', b'bar'), + (b'foo/bar/something', b'foo/bar/*'), +] + +NEGATIVE_MATCH_TESTS = [ + (b'foo.c', b'foo.[dh]'), + (b'foo/foo.c', b'/foo.c'), + (b'foo/foo.c', b'/*.c'), + (b'foo/bar/', b'/bar/'), + (b'foo/bar/', b'foo/bar/*'), + (b'foo/bar', b'foo?bar'), +] + +TRANSLATE_TESTS = [ + (b'*.c', b'(?ms)(.*/)?[^/]*\\.c/?\\Z'), + (b'foo.c', b'(?ms)(.*/)?foo\\.c/?\\Z'), + (b'/*.c', b'(?ms)[^/]*\\.c/?\\Z'), + (b'/foo.c', b'(?ms)foo\\.c/?\\Z'), + (b'foo.c', b'(?ms)(.*/)?foo\\.c/?\\Z'), + (b'foo.[ch]', b'(?ms)(.*/)?foo\\.[ch]/?\\Z'), + (b'bar/', b'(?ms)(.*/)?bar\\/\\Z'), + (b'foo/**', b'(?ms)foo(/.*)?/?\\Z'), + (b'foo/**/blie.c', b'(?ms)foo(/.*)?\\/blie\\.c/?\\Z'), + (b'**/bla.c', b'(?ms)(.*/)?bla\\.c/?\\Z'), + (b'foo/**/bar', b'(?ms)foo(/.*)?\\/bar/?\\Z'), + (b'foo/bar/*', b'(?ms)foo\\/bar\\/[^/]+/?\\Z'), + (b'/foo\\[bar\\]', b'(?ms)foo\\[bar\\]/?\\Z'), + (b'/foo[bar]', b'(?ms)foo[bar]/?\\Z'), + (b'/foo[0-9]', b'(?ms)foo[0-9]/?\\Z'), +] + + +@pytest.mark.usefixtures('fs') +class TestIgnoreFiles: + def test_translate(self) -> None: + for pattern, regex in TRANSLATE_TESTS: + if re.escape(b'/') == b'/': + regex = regex.replace(b'\\/', b'/') + assert translate(pattern) == regex, ( + f'orig pattern: {pattern!r}, regex: {translate(pattern)!r}, expected: {regex!r}' + ) + + def test_read_file(self) -> None: + f = BytesIO( + b""" +# a comment +\x20\x20 +# and an empty line: + +\\#not a comment +!negative +with trailing whitespace +with escaped trailing whitespace\\ +""" # noqa: W291 (Trailing whitespace) + ) + assert list(read_ignore_patterns(f)) == [ + b'\\#not a comment', + b'!negative', + b'with trailing whitespace', + b'with escaped trailing whitespace ', + ] + + def test_match_patterns_positive(self) -> None: + for path, pattern in POSITIVE_MATCH_TESTS: + assert match_pattern(path, pattern), f'path: {path!r}, pattern: {pattern!r}' + + def test_match_patterns_negative(self) -> None: + for path, pattern in NEGATIVE_MATCH_TESTS: + assert not match_pattern(path, pattern), f'path: {path!r}, pattern: {pattern!r}' + + def test_ignore_filter_inclusion(self) -> None: + ignore_filter = IgnoreFilter([b'a.c', b'b.c']) + assert ignore_filter.is_ignored(b'a.c') + assert ignore_filter.is_ignored(b'c.c') is None + assert list(ignore_filter.find_matching(b'a.c')) == [Pattern(b'a.c')] + assert list(ignore_filter.find_matching(b'c.c')) == [] + + def test_ignore_filter_exclusion(self) -> None: + ignore_filter = IgnoreFilter([b'a.c', b'b.c', b'!c.c']) + assert not ignore_filter.is_ignored(b'c.c') + assert ignore_filter.is_ignored(b'd.c') is None + assert list(ignore_filter.find_matching(b'c.c')) == [Pattern(b'!c.c')] + assert list(ignore_filter.find_matching(b'd.c')) == [] + + def test_ignore_filter_manager(self, fs: 'FakeFilesystem') -> None: + # Prepare sample ignore patterns + fs.create_file('/path/to/repo/.gitignore', contents=b'/foo/bar\n/dir2\n/dir3/\n') + fs.create_file('/path/to/repo/dir/.gitignore', contents=b'/blie\n') + fs.create_file('/path/to/repo/.git/info/exclude', contents=b'/excluded\n') + + m = IgnoreFilterManager.build('/path/to/repo') + + assert m.is_ignored('dir/blie') + assert m.is_ignored(os.path.join('dir', 'bloe')) is None + assert m.is_ignored('dir') is None + assert m.is_ignored(os.path.join('foo', 'bar')) + assert m.is_ignored(os.path.join('excluded')) + assert m.is_ignored(os.path.join('dir2', 'fileinignoreddir')) + assert not m.is_ignored('dir3') + assert m.is_ignored('dir3/') + assert m.is_ignored('dir3/bla') + + def test_nested_gitignores(self, fs: 'FakeFilesystem') -> None: + fs.create_file('/path/to/repo/.gitignore', contents=b'/*\n!/foo\n') + fs.create_file('/path/to/repo/foo/.gitignore', contents=b'/bar\n') + fs.create_file('/path/to/repo/foo/bar', contents=b'IGNORED') + + m = IgnoreFilterManager.build('/path/to/repo') + assert m.is_ignored('foo/bar') + + def test_load_ignore_ignore_case(self, fs: 'FakeFilesystem') -> None: + fs.create_file('/path/to/repo/.gitignore', contents=b'/foo/bar\n/dir\n') + + m = IgnoreFilterManager.build('/path/to/repo', ignore_case=True) + assert m.is_ignored(os.path.join('dir', 'blie')) + assert m.is_ignored(os.path.join('DIR', 'blie')) + + def test_ignored_contents(self, fs: 'FakeFilesystem') -> None: + fs.create_file('/path/to/repo/.gitignore', contents=b'a/*\n!a/*.txt\n') + + m = IgnoreFilterManager.build('/path/to/repo') + assert m.is_ignored('a') is None + assert m.is_ignored('a/') is None + assert not m.is_ignored('a/b.txt') + assert m.is_ignored('a/c.dat') diff --git a/tests/utils/test_path_utils.py b/tests/utils/test_path_utils.py index d5c9e3ae..eaf67f65 100644 --- a/tests/utils/test_path_utils.py +++ b/tests/utils/test_path_utils.py @@ -4,31 +4,31 @@ from tests.conftest import TEST_FILES_PATH -def test_is_sub_path_both_paths_are_same(): +def test_is_sub_path_both_paths_are_same() -> None: path = os.path.join(TEST_FILES_PATH, 'hello') sub_path = os.path.join(TEST_FILES_PATH, 'hello') assert is_sub_path(path, sub_path) is True -def test_is_sub_path_path_is_not_subpath(): +def test_is_sub_path_path_is_not_subpath() -> None: path = os.path.join(TEST_FILES_PATH, 'hello') sub_path = os.path.join(TEST_FILES_PATH, 'hello.txt') assert is_sub_path(path, sub_path) is False -def test_is_sub_path_path_is_subpath(): +def test_is_sub_path_path_is_subpath() -> None: path = os.path.join(TEST_FILES_PATH, 'hello') sub_path = os.path.join(TEST_FILES_PATH, 'hello', 'random.txt') assert is_sub_path(path, sub_path) is True -def test_is_sub_path_path_not_exists(): +def test_is_sub_path_path_not_exists() -> None: path = os.path.join(TEST_FILES_PATH, 'goodbye') sub_path = os.path.join(TEST_FILES_PATH, 'hello', 'random.txt') assert is_sub_path(path, sub_path) is False -def test_is_sub_path_subpath_not_exists(): +def test_is_sub_path_subpath_not_exists() -> None: path = os.path.join(TEST_FILES_PATH, 'hello', 'random.txt') sub_path = os.path.join(TEST_FILES_PATH, 'goodbye') assert is_sub_path(path, sub_path) is False diff --git a/tests/utils/test_string_utils.py b/tests/utils/test_string_utils.py new file mode 100644 index 00000000..8c94fceb --- /dev/null +++ b/tests/utils/test_string_utils.py @@ -0,0 +1,7 @@ +from cycode.cli.utils.string_utils import shortcut_dependency_paths + + +def test_shortcut_dependency_paths_list_single_dependencies() -> None: + dependency_paths = 'A, A -> B, A -> B -> C' + expected_result = 'A\nA -> B\nA -> ... -> C' + assert shortcut_dependency_paths(dependency_paths) == expected_result diff --git a/tests/utils/test_url_utils.py b/tests/utils/test_url_utils.py new file mode 100644 index 00000000..f7f6b6b0 --- /dev/null +++ b/tests/utils/test_url_utils.py @@ -0,0 +1,80 @@ +from cycode.cli.utils.url_utils import sanitize_repository_url + + +def test_sanitize_repository_url_with_token() -> None: + """Test that PAT tokens are removed from HTTPS URLs.""" + url = 'https://token@github.com/user/repo.git' + expected = 'https://github.com/user/repo.git' + assert sanitize_repository_url(url) == expected + + +def test_sanitize_repository_url_with_username_and_token() -> None: + """Test that username and token are removed from HTTPS URLs.""" + url = 'https://user:token@github.com/user/repo.git' + expected = 'https://github.com/user/repo.git' + assert sanitize_repository_url(url) == expected + + +def test_sanitize_repository_url_with_port() -> None: + """Test that URLs with ports are handled correctly.""" + url = 'https://token@github.com:443/user/repo.git' + expected = 'https://github.com:443/user/repo.git' + assert sanitize_repository_url(url) == expected + + +def test_sanitize_repository_url_ssh_format() -> None: + """Test that SSH URLs are returned as-is (no credentials in URL format).""" + url = 'git@github.com:user/repo.git' + assert sanitize_repository_url(url) == url + + +def test_sanitize_repository_url_ssh_protocol() -> None: + """Test that ssh:// URLs are returned as-is.""" + url = 'ssh://git@github.com/user/repo.git' + assert sanitize_repository_url(url) == url + + +def test_sanitize_repository_url_no_credentials() -> None: + """Test that URLs without credentials are returned unchanged.""" + url = 'https://github.com/user/repo.git' + assert sanitize_repository_url(url) == url + + +def test_sanitize_repository_url_none() -> None: + """Test that None input returns None.""" + assert sanitize_repository_url(None) is None + + +def test_sanitize_repository_url_empty_string() -> None: + """Test that empty string is returned as-is.""" + assert sanitize_repository_url('') == '' + + +def test_sanitize_repository_url_gitlab() -> None: + """Test that GitLab URLs are sanitized correctly.""" + url = 'https://oauth2:token@gitlab.com/user/repo.git' + expected = 'https://gitlab.com/user/repo.git' + assert sanitize_repository_url(url) == expected + + +def test_sanitize_repository_url_bitbucket() -> None: + """Test that Bitbucket URLs are sanitized correctly.""" + url = 'https://x-token-auth:token@bitbucket.org/user/repo.git' + expected = 'https://bitbucket.org/user/repo.git' + assert sanitize_repository_url(url) == expected + + +def test_sanitize_repository_url_with_path_and_query() -> None: + """Test that URLs with paths, query params, and fragments are preserved.""" + url = 'https://token@github.com/user/repo.git?ref=main#section' + expected = 'https://github.com/user/repo.git?ref=main#section' + assert sanitize_repository_url(url) == expected + + +def test_sanitize_repository_url_invalid_url() -> None: + """Test that invalid URLs are returned as-is (graceful degradation).""" + # This should not raise an exception, but return the original + url = 'not-a-valid-url' + result = sanitize_repository_url(url) + # Should return original since parsing fails + assert result == url