diff --git a/.allstar/binary_artifacts.yaml b/.allstar/binary_artifacts.yaml new file mode 100644 index 0000000000..ed03308c1a --- /dev/null +++ b/.allstar/binary_artifacts.yaml @@ -0,0 +1,5 @@ +# Linkage Checker has test to examine JAR files and class files +ignorePaths: +- enforcer-rules/src/test/resources/dummy-0.0.1.jar +- example-problems/gradle-project/build/classes/java/main/App.class + diff --git a/.github/release-please.yml b/.github/release-please.yml new file mode 100644 index 0000000000..fa603d47eb --- /dev/null +++ b/.github/release-please.yml @@ -0,0 +1,11 @@ +handleGHRelease: true +manifest: true +bumpMinorPreMajor: true +packageName: gcp-lts-bom +branches: + - branch: 9.0.x-lts + - branch: 8.0.x-lts + - branch: 7.0.x-lts + - branch: 6.0.x-lts + - branch: protobuf-4.x-rc + releaseType: java-yoshi diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8adc5980e1..051d58f0e7 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -11,19 +11,24 @@ jobs: matrix: java: [8, 11] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: actions/setup-java@v1 with: java-version: ${{matrix.java}} - name: Get current date id: date - run: echo "::set-output name=date::$(date +'%Y-%m-%d' --utc)" - - uses: actions/cache@v2 + run: echo "date=$(date +'%Y-%m-%d' --utc)" >> "$GITHUB_OUTPUT" + - uses: actions/cache@v4 id: mvn-cache with: path: ~/.m2/repository key: ${{ runner.os }}-maven-unified-${{ steps.date.outputs.date }} - run: java -version - - run: ./mvnw -B -e -ntp install + # The http connection settings avoid Maven's HTTP connection reset in GitHub Actions + # https://github.com/actions/virtual-environments/issues/1499#issuecomment-689467080 + - run: | + ./mvnw -B -e -ntp install \ + -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false \ + -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 - run: cd gradle-plugin && ./gradlew build publishToMavenLocal diff --git a/.gitignore b/.gitignore index 09f3031d79..f40f7ab851 100644 --- a/.gitignore +++ b/.gitignore @@ -18,4 +18,4 @@ example-problems/*/gradle example-problems/*/gradlew example-problems/*/gradlew.bat gradle-plugin/.gradle/ - +.mvn/wrapper/maven-wrapper.jar diff --git a/.mvn/wrapper/maven-wrapper.jar b/.mvn/wrapper/maven-wrapper.jar deleted file mode 100644 index 2cc7d4a55c..0000000000 Binary files a/.mvn/wrapper/maven-wrapper.jar and /dev/null differ diff --git a/.release-please-manifest.json b/.release-please-manifest.json new file mode 100644 index 0000000000..bfc6a2aeb1 --- /dev/null +++ b/.release-please-manifest.json @@ -0,0 +1,5 @@ + + { + "dependencies": "1.5.15", + "boms/cloud-lts-bom": "9.0.1" + } diff --git a/CHANGELOG.md b/CHANGELOG.md index 52b8b30146..b8ec66ca97 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Linkage Checker Enforcer Rule and Linkage Monitor Change Log +## 1.5.12 +* Fixed the bug in the Gradle plugin that affected artifacts with "pom" packaging ([#2196]( + https://github.com/GoogleCloudPlatform/cloud-opensource-java/pull/2196)) + ## 1.5.11 * The Gradle plugin omits duplicate dependency paths when printing the locatin of problematic artifacts ([#2188](https://github.com/GoogleCloudPlatform/cloud-opensource-java/pull/2188)). diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 84296313e5..11257a8cdb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,6 +1,10 @@ # How to become a contributor and submit your own code -Pull requests are welcome. +The code in this repository is only intended to be used as part of the Google Cloud SDK build/test/release infrastructure, and it is not a supported Google product. Please make sure you understand the purpose of this repo before contributing. If you still would like to contribute, please follow the guidelines below before opening an issue or a PR: +1. Ensure the issue was not already reported. +2. Open a new issue if you are unable to find an existing issue addressing your problem. Make sure to include a title and clear description, as much relevant information as possible, and a code sample or an executable test case demonstrating the expected behavior that is not occurring. +3. Discuss the priority and potential solutions with the maintainers in the issue. The maintainers would review the issue and add a label "Accepting Contributions" once the issue is ready for accepting contributions. +4. Open a PR only if the issue is labeled with "Accepting Contributions", ensure the PR description clearly describes the problem and solution. Note that an open PR without an "Accepting Contributions" issue will not be accepted. ## Contributor License Agreements diff --git a/README.md b/README.md index 6b0ccbdd8d..a4c78f000d 100644 --- a/README.md +++ b/README.md @@ -3,14 +3,6 @@ This project explores common infrastructure and best practices for open source Java projects for the Google Cloud Platform (GCP). -# Google Cloud Platform Java Dependency Dashboard - -[Google Cloud Platform Java Dependency Dashboard]( -https://storage.googleapis.com/cloud-opensource-java-dashboard/com.google.cloud/libraries-bom/snapshot/index.html) -(runs daily; work in progress) shows multiple checks on the consistency among -Google Cloud Java libraries. For manually generating the dashboard, see -[its README](./dashboard/README.md). - # Google Best Practices for Java Libraries [Google Best Practices for Java Libraries](https://googlecloudplatform.github.io/cloud-opensource-java/) @@ -39,10 +31,12 @@ https://github.com/GoogleCloudPlatform/cloud-opensource-java/wiki/Linkage-Checke # GCP Libraries BOM -The [GCP Libraries BOM](https://github.com/GoogleCloudPlatform/cloud-opensource-java/wiki/The-Google-Cloud-Platform-Libraries-BOM) is a Bill-of-Materials (BOM) that +The [GCP Libraries BOM](https://cloud.google.com/java/docs/bom) is a Bill-of-Materials (BOM) that provides consistent versions of Google Cloud Java libraries that work together without linkage errors. +This has moved to https://github.com/googleapis/java-cloud-bom/tree/main/libraries-bom. + # Development This project is built using _Maven_. @@ -57,3 +51,8 @@ This project is built using _Maven_. 1. Clone the project to a local directory using `git clone git@github.com:GoogleCloudPlatform/cloud-opensource-java.git`. + +# Disclaimer + +This is not an officially supported Google product. + diff --git a/boms/cloud-lts-bom/RELEASING.md b/boms/cloud-lts-bom/RELEASING.md deleted file mode 100644 index 828499c67a..0000000000 --- a/boms/cloud-lts-bom/RELEASING.md +++ /dev/null @@ -1,70 +0,0 @@ -# LTS BOM Release - -## Prerequisites - -For the prerequisites, see [Libraries BOM release's Prerequisites section]( -../cloud-oss-bom/RELEASING.md). - -## Steps - -All on your corp desktop: - -1. Run gcert if you have not done so in the last twelve hours or so. - -2. If this is a major version release, create a LTS release branch `N.0.x-lts`, where -N is the release number. - -For example: - -``` -$ git checkout -b 2.0.x-lts origin/master -$ git push --set-upstream origin 2.0.x-lts -``` - -If this is a patch version release, the LTS release branch already exists. -No action required for this step #2. - -3. Run `release.sh` with `lts` argument in -the `cloud-opensource-java` directory: - -``` -$ ./scripts/release.sh lts -``` - -With the `lts` argument, this script creates a pull request and that bumps the patch version of -the LTS release branch (not the master branch). -Ask a teammate to review and approve the PR. - -Behind the scenes, the release script creates a release branch based on the -corresponding LTS release branch. -For example, when you run the release script with argument `./scripts/release.sh lts 5.0.3`, -it creates a release branch `5.0.3-lts` based on the LTS release branch `5.0.x-lts`. - -The script also initiates the Rapid release workflow. - -4. After you finish the release, if this is a major version release, bump the major version of the - BOM (`boms/cloud-lts-bom/pom.xml`) in the master branch. - -## OSSRH - -The Rapid workflow uploads the artifact to OSSRH staging repository. - -[Instructions for releasing from OSSRH are on the internal team -site](https://g3doc.corp.google.com/company/teams/cloud-java/tools/developers/releasing.md#verify-and-release). - - -## Making changes for a patch release - -Changes for a patch release should be committed to LTS release branches (not to the master branch). - -We maintain multiple branches for the LTS BOM releases. -Let's call the branches _LTS release branches_. -An LTS release branch has a name "N.0.x-lts", where N is the major release version number, -such as `1.0.x-lts` and `2.0.x-lts`. -In this GitHub repository, the branches are [configured as protected]( -https://github.com/GoogleCloudPlatform/cloud-opensource-java/settings/branches), -so that we do not accidentally push changes into them. - -Any changes for a patch release to the LTS BOM should be merged to one of the LTS release branches. -This practice enables us to release a patch version of one of the old versions of the BOM, -without using the master branch (or _HEAD_). diff --git a/boms/cloud-lts-bom/pom.xml b/boms/cloud-lts-bom/pom.xml index 90b4a4cf15..a803970ccb 100644 --- a/boms/cloud-lts-bom/pom.xml +++ b/boms/cloud-lts-bom/pom.xml @@ -7,9 +7,8 @@ com.google.cloud gcp-lts-bom - 2.0.0-SNAPSHOT + 9.0.2-SNAPSHOT pom - Google Cloud Long Term Support BOM Google Cloud Long Term Support BOM https://github.com/GoogleCloudPlatform/cloud-opensource-java @@ -47,32 +46,57 @@ UTF-8 - 2.30.0 - - 30.1.1-jre - 1.8.1 - 3.16.0 - 1.39.2-sp.1 - 1.36.2 - - 1.64.0-sp.1 - 0.81.0-sp.1 - 1.31.3-sp.1 - 1.10.1-sp.1 - 0.25.2-sp.1 - 1.31.4-sp.1 - 3.3.3-sp.1 - 2.1.0-sp.1 - 1.3.0-sp.1 - 1.111.0-sp.1 - 1.127.12-sp.1 - 1.22.0-sp.1 - 1.20.0-sp.1 - 1.113.14-sp.1 - 1.6.4-sp.1 - 1.9.89 - v3-rev20210429-1.31.0 + + + 33.4.0-jre + 1.11.0 + 3.25.8 + 1.71.0 + 1.47.0 + 1.39.0 + 1.36.0 + 2.7.2 + + 2.67.0 + 2.50.0 + 2.57.0 + 2.58.0 + + + 2.67.0 + 2.67.0 + 3.65.0 + 1.64.0 + 2.67.0 + 1.66.0 + 2.64.0 + 2.64.0 + 2.64.0 + 2.64.0 + 3.62.0 + 3.59.0 + 2.64.0 + 2.64.0 + v3-rev20250602-2.0.0 + 2.0.34 + 3.0.9 + 2.51.0 + + v2-rev20250511-2.0.0 + 3.15.0 + 2.60.0 + 2.29.1 + 3.22.5 + 1.140.1 + + 1.122.1 + 6.95.1 + 2.53.0 + + + 2.67.0 + + 2.15.3 @@ -84,34 +108,11 @@ + + - org.apache.beam - beam-sdks-java-core - ${beam.version} - - - org.apache.beam - beam-sdks-java-extensions-google-cloud-platform-core - ${beam.version} - - - org.apache.beam - beam-runners-google-cloud-dataflow-java - ${beam.version} - - - org.apache.beam - beam-sdks-java-io-google-cloud-platform - ${beam.version} - - - com.google.cloud - google-cloud-bigquery - ${google.cloud.bigquery.version} - - - + com.google.guava guava ${guava.version} @@ -119,7 +120,7 @@ com.google.auto.value auto-value-annotations - ${google.autovalue.version} + ${autovalue.version} com.google.protobuf @@ -130,6 +131,32 @@ com.google.protobuf protobuf-java-util ${protobuf.version} + + + com.google.code.gson + gson + + + + + com.google.http-client + google-http-client + ${google-http-client.version} + + + com.google.api + api-common + ${api-common.version} + + + com.google.oauth-client + google-oauth-client + ${google-oauth-client.version} + + + com.google.auth + google-auth-library-oauth2-http + ${google-auth-library.version} @@ -203,99 +230,366 @@ grpc-testing ${io.grpc.version} - - com.google.appengine - appengine-api-1.0-sdk - ${appengine.api.1.0.sdk.version} - com.google.api-client google-api-client - ${google.api.client.version} + ${google-api-client.version} com.google.api-client google-api-client-appengine - ${google.api.client.version} + ${google-api-client.version} - com.google.auth - google-auth-library-oauth2-http - ${google.auth.library.version} + com.google.api.grpc + proto-google-common-protos + ${proto-google-common-protos.version} - com.google.oauth-client - google-oauth-client - ${google.oauth.client.version} + com.google.api + gax + ${gax.version} + + + com.google.api + gax-grpc + ${gax.version} + + + com.google.api + gax-httpjson + ${gax.version} + + + com.google.cloud + google-cloud-core + ${google-cloud-core.version} + + + + + com.google.appengine + appengine-api-1.0-sdk + ${appengine-api-1.0-sdk.version} + + + com.google.appengine + appengine-testing + ${appengine-api-1.0-sdk.version} + + + com.google.cloud + google-iam-admin-bom + ${google-iam-admin.version} + pom + import + + + com.google.cloud + google-cloud-iamcredentials-bom + ${google-cloud-iamcredentials.version} + pom + import + + + com.google.apis + google-api-services-androidpublisher + ${google-api-services-androidpublisher.version} + + + com.google.cloud + google-cloud-bigquery + ${google-cloud-bigquery.version} + + + com.google.apis + google-api-services-bigquery + ${google-api-services-bigquery.version} + + + com.google.cloud + google-cloud-bigquerystorage-bom + ${google-cloud-bigquerystorage.version} + pom + import com.google.cloud google-cloud-storage - ${google.cloud.storage.version} + ${google-cloud-storage.version} com.google.cloud - google-cloud-bigtable - ${google.cloud.bigtable.version} + google-cloud-trace-bom + ${google-cloud-trace.version} + pom + import - com.google.cloud.bigtable - bigtable-hbase-beam - ${bigtable-hbase-beam.version} + com.google.cloud + google-cloud-translate-bom + ${google-cloud-translate.version} + pom + import com.google.cloud - google-cloud-trace - ${google.cloud.trace.version} + google-cloud-monitoring-bom + ${google-cloud-monitoring.version} + pom + import com.google.cloud - google-cloud-monitoring - ${google.cloud.monitoring.version} + google-cloud-spanner + ${google-cloud-spanner.version} + + + com.google.api.grpc + proto-google-cloud-spanner-admin-database-v1 + ${google-cloud-spanner.version} + + + com.google.api.grpc + proto-google-cloud-spanner-v1 + ${google-cloud-spanner.version} + + + com.google.cloud + google-cloud-tasks-bom + ${google-cloud-tasks.version} + pom + import + + + com.google.cloud + + google-cloud-bigtable + ${google-cloud-bigtable.version} + + + com.google.api.grpc + proto-google-cloud-bigtable-v2 + ${google-cloud-bigtable.version} com.google.cloud google-cloud-pubsub - ${google.cloud.pubsub.version} + ${google-cloud-pubsub.version} + + + com.google.api.grpc + proto-google-cloud-pubsub-v1 + ${proto-google-cloud-pubsub-v1.version} + + + com.google.cloud + google-cloud-datastore-bom + ${google-cloud-datastore.version} + pom + import com.google.cloud.datastore datastore-v1-proto-client - ${datastore.v1.proto.client.version} + ${google-cloud-datastore.version} com.google.cloud - google-cloud-spanner - ${google.cloud.spanner.version} + google-cloud-service-usage-bom + ${google-cloud-service-usage.version} + pom + import - com.google.apis - google-api-services-androidpublisher - ${androidpublisher.version} + com.google.cloud + google-cloud-container-bom + ${google-cloud-container.version} + pom + import + + + com.google.cloud + google-cloud-orchestration-airflow-bom + ${google-cloud-orchestration-airflow.version} + pom + import + + + com.google.cloud + google-cloud-resourcemanager-bom + ${google-cloud-resourcemanager.version} + pom + import + + + com.google.cloud + google-cloud-redis-bom + ${google-cloud-redis.version} + pom + import + + + com.google.cloud + google-cloud-logging-bom + ${google-cloud-logging.version} + pom + import + + + com.google.cloud + google-cloud-secretmanager-bom + ${google-cloud-secretmanager.version} + pom + import + + + com.google.cloud + google-cloud-kms-bom + ${google-cloud-kms.version} + pom + import + + + com.google.cloud + google-cloud-vision-bom + ${google-cloud-vision.version} + pom + import - - com.google.api - api-common - ${api.common.version} + com.google.cloud.bigdataoss + gcs-connector + ${gcs-connector.version} - com.google.api - gax - ${gax.version} + com.google.cloud.bigdataoss + gcsio + ${gcs-connector.version} - com.google.api - gax-grpc - ${gax.version} + com.google.cloud.bigdataoss + util + ${gcs-connector.version} + + - com.google.http-client - google-http-client - ${http.version} + org.apache.beam + beam-sdks-java-core + ${beam.version} + + + com.google.protobuf + protobuf-java + + + com.google.protobuf + protobuf-java-util + + + + + org.apache.beam + beam-sdks-java-extensions-google-cloud-platform-core + ${beam.version} + + + com.google.protobuf + protobuf-java + + + com.google.protobuf + protobuf-java-util + + + + + org.apache.beam + beam-runners-google-cloud-dataflow-java + ${beam.version} + + + com.google.protobuf + protobuf-java + + + com.google.protobuf + protobuf-java-util + + + + + org.apache.beam + beam-sdks-java-io-google-cloud-platform + ${beam.version} + + + com.google.protobuf + protobuf-java + + + com.google.protobuf + protobuf-java-util + + + + + + + com.google.cloud.bigtable + bigtable-hbase-beam + ${bigtable-hbase-beam.version} + + + release + + + performRelease + + + + + + org.apache.maven.plugins + maven-gpg-plugin + 3.2.7 + + + sign-artifacts + verify + + sign + + + + --pinentry-mode + loopback + + + + + + + org.apache.maven.plugins + maven-source-plugin + 3.3.1 + + + attach-sources + + jar-no-fork + + + + + + + + diff --git a/boms/cloud-lts-bom/versions.txt b/boms/cloud-lts-bom/versions.txt new file mode 100644 index 0000000000..530d658838 --- /dev/null +++ b/boms/cloud-lts-bom/versions.txt @@ -0,0 +1,4 @@ +# Format: +# module:released-version:current-version + +gcp-lts-bom:9.0.1:9.0.2-SNAPSHOT diff --git a/boms/cloud-oss-bom/RELEASING.md b/boms/cloud-oss-bom/RELEASING.md deleted file mode 100644 index 1ecc53dd67..0000000000 --- a/boms/cloud-oss-bom/RELEASING.md +++ /dev/null @@ -1,152 +0,0 @@ -# Cloud Libraries BOM Release - - -## Prerequisites - -(Do not need to be repeated for each release.) - -* Install the [`gh`](https://github.com/cli/cli) -tool if you not previously done so. - - * Run `gh auth login` to register your desktop with github. - -* Clone this repository onto your corp desktop, Ubiquity instance, or CloudTop. Do not use a laptop or personal machine as the release requires google3 access. - -* Install and configure the [repo tool](https://github.com/googleapis/github-repo-automation). - -## Decide the release version - -To decide the release version, check the changes since the last release. -(This step ensures that you do not miss expected libraries upgrades.) -For example, if the last BOM release was version 16.4.0, then run the following command -to see the difference. - -``` -git diff v16.4.0-bom -- boms/cloud-oss-bom/pom.xml -``` - -If the difference includes the google-cloud-bom version, then check the change in the release note -at https://github.com/googleapis/java-cloud-bom/releases as well. - -From these changes in the content of the Libraries BOM, -determine the release version by the following logic: - -- If there is at least one major version bump among the changes, it's a major version bump. -- If there is at least one minor version bump (no major version change), it's a minor version - bump. -- If there are only patch version bumps (no major or minor version change), it's a patch version - bump. - -We use the release version for `release.sh` in the next steps. - -## Steps - -All on your corp desktop: - -1. Run gcert if you have not done so in the last twelve hours or so. - -2. Run `release.sh` with `bom` argument in -the `cloud-opensource-java` directory: - -``` -$ ./scripts/release.sh bom -``` - -You might see this message: - -``` -Notice: authentication required -Press Enter to open github.com in your browser... -``` - -Do it. This grants the script permission to create a PR for you on Github. - -Ask a teammate to review and approve the PR. - -If you want the script to stop asking your username and password for every invocation, -run `git config credential.helper store`. - -### Build the release binary with Rapid (Legacy web UI) - -The [instructions for the Rapid build are on the internal team -site](https://g3doc.corp.google.com/company/teams/cloud-java/tools/developers/releasing.md#run-the-rapid-workflow). - -## OSSRH - -[Instructions for releasing from OSSRH are on the internal team -site](https://g3doc.corp.google.com/company/teams/cloud-java/internal/g3doc/tools/releasing.md?cl=head). - -## Update the docs - -Several docs in this and other repositories need to be updated once the -new release is available on Maven Central. - -* Send pull requests that change the version in these documents: - * https://github.com/GoogleCloudPlatform/cloud-opensource-java/wiki/The-Google-Cloud-Platform-Libraries-BOM - (no PR required) - * https://github.com/googleapis/google-http-java-client/blob/master/docs/setup.md - * https://github.com/googleapis/google-cloud-java/blob/master/TROUBLESHOOTING.md -* Ask a code owner for java-docs-samples to merge the dependabot PR - that updates libraries-bom in https://github.com/GoogleCloudPlatform/java-docs-samples/pulls -* Use the repo tool to approve renovatebot updates for libraries-bom in the individual clients: - * `$ repo list --title .*v16.4.0` - * Verify that the listed PRs look correct and don't include anything you're not ready to merge. - * `$ repo approve --title .*v16.4.0` - * `$ repo --title .*v16.4.0 tag automerge` -* Manually edit and update any pom.xml files in https://github.com/GoogleCloudPlatform/java-docs-samples that dependabot missed -* In google3 run: - * `$ scripts/update_docs.sh ` - * For example, `$ scripts/update_docs.sh 16.3.0 16.4.0` - * When asked whether to add changes to the first CL, answer yes. - * Sanity check the CL and send it for review. - * Submit on approval -* Search for libraries-bom in google3 to find any internal references (typically cloudsite and devsite) that still need to be updated. - -## Retrying a failed release - -If the Github steps succeed—PR created, version tagged, etc.—but the Rapid release fails, you can -run this command from a g4 client to retry the Rapid build without going all the way -back to the beginning: - -``` -$ blaze run java/com/google/cloud/java/tools:ReleaseRapidProject -- \ - --project_name=cloud-java-tools-cloud-opensource-java-bom-kokoro-release \ - --committish=v${VERSION}-bom -``` - -## Deleting a release - -Occasionally you need to clean up after an aborted release, typically because the release script had -problems. If so: - -1. Delete the release branch on Github. - -2. Run `scripts/cancel_release.sh ` - -3. If the release got as far as uploading a binary to Nexus before you cancelled, then -login to OSSRH and drop the release. - - -The `cancel_release.sh` script performs these steps: - - -1. Fetch the tags in your local client: - - ``` - $ git fetch --tags --force - ``` - -2. Delete the tag locally: - - ``` - $ git tag -d v2.6.0-bom - Deleted tag 'v2.6.0-bom' (was 3b96602) - ``` - -2. Push the deleted tag: - - ``` - $ git push origin :v2.6.0-bom - To github.com:GoogleCloudPlatform/cloud-opensource-java.git - - [deleted] v2.6.0-bom - ``` diff --git a/boms/cloud-oss-bom/pom.xml b/boms/cloud-oss-bom/pom.xml deleted file mode 100644 index bf535b6550..0000000000 --- a/boms/cloud-oss-bom/pom.xml +++ /dev/null @@ -1,280 +0,0 @@ - - - 4.0.0 - - com.google.cloud - libraries-bom - 21.0.1-SNAPSHOT - pom - - Google Cloud Platform Supported Libraries - A compatible set of Google Cloud open source libraries. - https://github.com/GoogleCloudPlatform/cloud-opensource-java/wiki/The-Google-Cloud-Platform-Libraries-BOM - - Google LLC - https://cloud.google.com - - 2019 - - - Elliotte Rusty Harold - - - - - https://github.com/GoogleCloudPlatform/cloud-opensource-java/issues - - - scm:git:git@github.com:GoogleCloudPlatform/cloud-opensource-java.git - scm:git:git@github.com:GoogleCloudPlatform/cloud-opensource-java.git - - https://github.com/GoogleCloudPlatform/cloud-opensource-java/boms/cloud-oss-bom - HEAD - - - - - The Apache License, Version 2.0 - http://www.apache.org/licenses/LICENSE-2.0.txt - - - - - UTF-8 - 30.1.1-jre - 0.159.0 - 2.0.5 - 1.39.0 - 1.39.2 - 3.17.3 - - 2.1.0 - 0.86.0 - 1.0.0 - 2.0.1 - 2.3.2 - 1.0.14 - - - - - sonatype-nexus-snapshots - Sonatype Nexus Snapshots - https://oss.sonatype.org/content/repositories/snapshots/ - - - - - - - - com.google.guava - guava - ${guava.version} - - - com.google.guava - guava-testlib - ${guava.version} - - - - - com.google.protobuf - protobuf-bom - ${protobuf.version} - pom - import - - - - - com.google.http-client - google-http-client - ${http.version} - - - com.google.http-client - google-http-client-android - ${http.version} - - - com.google.http-client - google-http-client-apache-v2 - ${http.version} - - - com.google.http-client - google-http-client-appengine - ${http.version} - - - com.google.http-client - google-http-client-gson - ${http.version} - - - com.google.http-client - google-http-client-jackson2 - ${http.version} - - - com.google.http-client - google-http-client-protobuf - ${http.version} - - - com.google.http-client - google-http-client-test - ${http.version} - - - com.google.http-client - google-http-client-xml - ${http.version} - - - - - io.grpc - grpc-alts - ${io.grpc.version} - - - io.grpc - grpc-api - ${io.grpc.version} - - - io.grpc - grpc-auth - ${io.grpc.version} - - - io.grpc - grpc-context - ${io.grpc.version} - - - io.grpc - grpc-core - ${io.grpc.version} - - - io.grpc - grpc-grpclb - ${io.grpc.version} - - - io.grpc - grpc-netty - ${io.grpc.version} - - - io.grpc - grpc-netty-shaded - ${io.grpc.version} - - - io.grpc - grpc-okhttp - ${io.grpc.version} - - - io.grpc - grpc-protobuf - ${io.grpc.version} - - - io.grpc - grpc-protobuf-lite - ${io.grpc.version} - - - io.grpc - grpc-services - ${io.grpc.version} - - - io.grpc - grpc-stub - ${io.grpc.version} - - - io.grpc - grpc-testing - ${io.grpc.version} - - - - - com.google.cloud - google-cloud-bom - ${google.cloud.java.version} - pom - import - - - - com.google.api - api-common - ${api-common.version} - - - com.google.api - gax - ${gax.version} - - - com.google.api - gax-grpc - ${gax.version} - - - com.google.api - gax-httpjson - ${gax.httpjson.version} - - - com.google.auth - google-auth-library-bom - ${auth.version} - pom - import - - - com.google.cloud - google-cloud-core-bom - ${google.cloud.core.version} - pom - import - - - com.google.api.grpc - proto-google-common-protos - ${common.protos.version} - - - com.google.api.grpc - grpc-google-common-protos - ${common.protos.version} - - - com.google.api.grpc - proto-google-iam-v1 - ${iam.protos.version} - - - com.google.api.grpc - grpc-google-iam-v1 - ${iam.protos.version} - - - - - - diff --git a/boms/integration-tests/pom.xml b/boms/integration-tests/pom.xml index f2f7a29a87..62fdd4792d 100644 --- a/boms/integration-tests/pom.xml +++ b/boms/integration-tests/pom.xml @@ -18,7 +18,7 @@ com.google.cloud.tools dependencies - 1.5.12-SNAPSHOT + 1.5.16-SNAPSHOT junit @@ -38,7 +38,8 @@ false - -Xms128m -Xmx2048m + -Xms128m -Xmx4g + false diff --git a/boms/integration-tests/src/test/java/com/google/cloud/BomContentTest.java b/boms/integration-tests/src/test/java/com/google/cloud/BomContentTest.java new file mode 100644 index 0000000000..682cf3119b --- /dev/null +++ b/boms/integration-tests/src/test/java/com/google/cloud/BomContentTest.java @@ -0,0 +1,272 @@ +/* + * Copyright 2019 Google LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud; + +import com.google.cloud.tools.opensource.classpath.ClassPathBuilder; +import com.google.cloud.tools.opensource.classpath.ClassPathEntry; +import com.google.cloud.tools.opensource.classpath.ClassPathResult; +import com.google.cloud.tools.opensource.classpath.DependencyMediation; +import com.google.cloud.tools.opensource.dependencies.Artifacts; +import com.google.cloud.tools.opensource.dependencies.Bom; +import com.google.cloud.tools.opensource.dependencies.DependencyPath; +import com.google.common.base.Joiner; +import com.google.common.base.Verify; +import com.google.common.collect.ImmutableList; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URL; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.eclipse.aether.artifact.Artifact; +import org.eclipse.aether.graph.Dependency; +import org.eclipse.aether.util.version.GenericVersionScheme; +import org.eclipse.aether.version.InvalidVersionSpecificationException; +import org.eclipse.aether.version.Version; +import org.eclipse.aether.version.VersionScheme; +import org.junit.Assert; +import org.junit.Test; + +/** + * Checks the content of the BOMs in this repository. When some artifacts are not available in Maven + * Central yet, use "-DdisableMavenCentralCheck=true" system property when running this test. + */ +public class BomContentTest { + private static VersionScheme versionScheme = new GenericVersionScheme(); + + // List of Maven dependency scopes that are visible to library users. For example "provided" scope + // dependencies do not appear in users' class path. + private static final ImmutableList dependencyScopesVisibleToUsers = + ImmutableList.of("compile", "runtime"); + + @Test + public void testLtsBom() throws Exception { + Path bomPath = Paths.get("..", "cloud-lts-bom", "pom.xml").toAbsolutePath(); + checkBom(bomPath); + } + + private void checkBom(Path bomPath) throws Exception { + Bom bom = Bom.readBom(bomPath); + + // Sometimes the artifacts are not yet available in Maven Central and only available in local + // Maven repository. Use this property in that case. + boolean disableMavenCentralCheck = + "true".equals(System.getProperty("disableMavenCentralCheck")); + + List artifacts = bom.getManagedDependencies(); + if (!disableMavenCentralCheck) { + for (Artifact artifact : artifacts) { + assertReachable(buildMavenCentralUrl(artifact)); + } + } + // Temporarily ignore due to inability to process exclusion statements + //assertNoDowngradeRule(bom); + assertUniqueClasses(artifacts); + assertBomIsImported(bom); + } + + private static String buildMavenCentralUrl(Artifact artifact) { + return "https://repo1.maven.org/maven2/" + + artifact.getGroupId().replace('.', '/') + + "/" + + artifact.getArtifactId() + + "/" + + artifact.getVersion() + + "/"; + } + + /** + * Asserts that the BOM only provides JARs which contains unique class names to the classpath. + */ + private static void assertUniqueClasses(List allArtifacts) + throws InvalidVersionSpecificationException, IOException { + + StringBuilder errorMessageBuilder = new StringBuilder(); + + ClassPathBuilder classPathBuilder = new ClassPathBuilder(); + ClassPathResult result = + classPathBuilder.resolve(allArtifacts, false, DependencyMediation.MAVEN); + + // A Map of every class name to its artifact ID. + HashMap fullClasspathMap = new HashMap<>(); + + for (ClassPathEntry classPathEntry : result.getClassPath()) { + Artifact currentArtifact = classPathEntry.getArtifact(); + + String artifactId = currentArtifact.getArtifactId(); + String groupId = currentArtifact.getGroupId(); + if (!groupId.contains("google") + || groupId.contains("com.google.android") + || groupId.contains("com.google.cloud.bigtable") + || artifactId.startsWith("proto-") + || artifactId.equals("protobuf-javalite") + || artifactId.startsWith("appengine-")) { + // Skip libraries that produce false positives. + // See: https://github.com/GoogleCloudPlatform/cloud-opensource-java/issues/2226 + continue; + } + + String artifactCoordinates = Artifacts.toCoordinates(currentArtifact); + + for (String className : classPathEntry.getFileNames()) { + if (className.contains("javax.annotation") + || className.contains("$") + || className.equals("com.google.cloud.location.LocationsGrpc") + || className.endsWith("package-info") + || className.endsWith("module-info")) { + // Ignore annotations, nested classes, and package-info files. + // Ignore module-info files. + // Ignore LocationsGrpc classes which are duplicated in generated grpc libraries. + continue; + } + + String previousArtifact = fullClasspathMap.get(className); + + if (previousArtifact != null) { + String msg = String.format( + "Duplicate class %s found in classpath. Found in artifacts %s and %s.\n", + className, + previousArtifact, + artifactCoordinates); + errorMessageBuilder.append(msg); + } else { + fullClasspathMap.put(className, artifactCoordinates); + } + } + } + + String error = errorMessageBuilder.toString(); + Assert.assertTrue( + "Failing test due to duplicate classes found on classpath:\n" + error, error.isEmpty()); + } + + private static void assertReachable(String url) throws IOException { + HttpURLConnection connection = (HttpURLConnection) new URL(url).openConnection(); + connection.setRequestMethod("HEAD"); + try { + Assert.assertEquals( + "Could not reach " + url, HttpURLConnection.HTTP_OK, connection.getResponseCode()); + } catch (IOException ex) { + Assert.fail("Could not reach " + url + "\n" + ex.getMessage()); + } + } + + /** + * Asserts that the members of the {@code bom} satisfy the no-downgrade rule. This rule means that + * the members have the highest versions among the dependencies of them. If there's a violation, + * users of the BOM would see our BOM downgrading certain dependencies. Downgrading a dependency + * is bad practice in general because newer versions have more features (classes and methods). + * + *

For example, if google-http-client 1.40.1 is in the BOM, then no other libraries in the BOM + * depend on the higher version of the google-http-client. + * + * @param bom the BOM to assert with this no-downgrade rule. + */ + private static void assertNoDowngradeRule(Bom bom) throws InvalidVersionSpecificationException { + List violations = new ArrayList<>(); + Map bomArtifacts = new HashMap<>(); + for (Artifact artifact : bom.getManagedDependencies()) { + bomArtifacts.put(Artifacts.makeKey(artifact), artifact); + } + + for (Artifact artifact : bom.getManagedDependencies()) { + violations.addAll(findNoDowngradeViolation(bomArtifacts, artifact)); + } + + String violationMessage = Joiner.on("\n").join(violations); + Assert.assertTrue(violationMessage, violations.isEmpty()); + } + + /** + * Returns messages describing the violation of the no-downgrade rule by {@code artifact} against + * the BOM containing {@code bomArtifacts}. An empty list if there is no violations. + */ + private static ImmutableList findNoDowngradeViolation( + Map bomArtifacts, Artifact artifact) + throws InvalidVersionSpecificationException { + ImmutableList.Builder violations = ImmutableList.builder(); + + ClassPathBuilder classPathBuilder = new ClassPathBuilder(); + ClassPathResult result = + classPathBuilder.resolve(ImmutableList.of(artifact), false, DependencyMediation.MAVEN); + for (ClassPathEntry entry : result.getClassPath()) { + Artifact transitiveDependency = entry.getArtifact(); + String key = Artifacts.makeKey(transitiveDependency); + Artifact bomArtifact = bomArtifacts.get(key); + if (bomArtifact == null) { + // transitiveDependency is not part of the BOM + continue; + } + + Version versionInBom = versionScheme.parseVersion(bomArtifact.getVersion()); + Version versionInTransitiveDependency = + versionScheme.parseVersion(transitiveDependency.getVersion()); + + if (versionInTransitiveDependency.compareTo(versionInBom) <= 0) { + // When versionInTransitiveDependency is less than or equal to versionInBom, it satisfies + // the no-downgrade rule. + continue; + } + + // Filter by scopes that are invisible to library users + ImmutableList dependencyPaths = result.getDependencyPaths(entry); + Verify.verify( + !dependencyPaths.isEmpty(), + "The class path entry should have at least one dependency path from the root"); + boolean dependencyVisibleToUsers = false; + for (DependencyPath dependencyPath : dependencyPaths) { + int length = dependencyPath.size(); + // As the root element is an empty node, the last element is at "length - 2". + Dependency dependency = dependencyPath.getDependency(length - 2); + if (dependencyScopesVisibleToUsers.contains(dependency.getScope())) { + dependencyVisibleToUsers = true; + break; + } + } + if (!dependencyVisibleToUsers) { + // For provided-scope dependencies, we don't have to worry about them because they don't + // appear in library users' class path. For example, appengine-api-1.0-sdk are used via + // provided scope. + continue; + } + + // A violation of the no-downgrade rule is found. + violations.add( + artifact + + " has a transitive dependency " + + transitiveDependency + + ". This is higher version than " + + bomArtifact + + " in the BOM. Example dependency path: " + + dependencyPaths.get(0)); + } + return violations.build(); + } + + private void assertBomIsImported(Bom bom) { + // BOMs must be declared as "import" type. Otherwise, the BOM users would see + // "google-cloud-XXX-bom" as an artifact declared in the BOM, not the content of it. + for (Artifact artifact : bom.getManagedDependencies()) { + String artifactId = artifact.getArtifactId(); + Assert.assertFalse( + artifactId + " must be declared with import type", artifactId.endsWith("-bom")); + } + } +} diff --git a/boms/integration-tests/src/test/java/com/google/cloud/MaximumLinkageErrorsTest.java b/boms/integration-tests/src/test/java/com/google/cloud/MaximumLinkageErrorsTest.java deleted file mode 100644 index d958f0ef12..0000000000 --- a/boms/integration-tests/src/test/java/com/google/cloud/MaximumLinkageErrorsTest.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright 2019 Google LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package com.google.cloud; - -import com.google.cloud.tools.opensource.classpath.LinkageChecker; -import com.google.cloud.tools.opensource.classpath.LinkageProblem; -import com.google.cloud.tools.opensource.dependencies.Bom; -import com.google.cloud.tools.opensource.dependencies.MavenRepositoryException; -import com.google.cloud.tools.opensource.dependencies.RepositoryUtility; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Sets; -import com.google.common.collect.Sets.SetView; -import java.io.IOException; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Optional; -import org.eclipse.aether.RepositoryException; -import org.junit.Assert; -import org.junit.Test; - -public class MaximumLinkageErrorsTest { - - @Test - public void testForNewLinkageErrors() - throws IOException, MavenRepositoryException, RepositoryException { - // Not using RepositoryUtility.findLatestCoordinates, which may return a snapshot version - String version = findLatestNonSnapshotVersion(); - String baselineCoordinates = "com.google.cloud:libraries-bom:" + version; - Bom baseline = Bom.readBom(baselineCoordinates); - - Path bomFile = Paths.get("../cloud-oss-bom/pom.xml"); - Bom bom = Bom.readBom(bomFile); - - ImmutableSet oldProblems = - LinkageChecker.create(baseline).findLinkageProblems(); - LinkageChecker checker = LinkageChecker.create(bom); - ImmutableSet currentProblems = checker.findLinkageProblems(); - - // This only tests for newly missing methods, not new invocations of - // previously missing methods. - SetView newProblems = Sets.difference(currentProblems, oldProblems); - - // Check that no new linkage errors have been introduced since the baseline - StringBuilder message = new StringBuilder("Baseline BOM: " + baselineCoordinates + "\n"); - if (!newProblems.isEmpty()) { - message.append("Newly introduced problems:\n"); - message.append(LinkageProblem.formatLinkageProblems(newProblems, null)); - Assert.fail(message.toString()); - } - } - - private String findLatestNonSnapshotVersion() throws MavenRepositoryException { - ImmutableList versions = - RepositoryUtility.findVersions( - RepositoryUtility.newRepositorySystem(), "com.google.cloud", "libraries-bom"); - ImmutableList versionsLatestFirst = versions.reverse(); - Optional highestNonsnapshotVersion = - versionsLatestFirst.stream().filter(version -> !version.contains("SNAPSHOT")).findFirst(); - if (!highestNonsnapshotVersion.isPresent()) { - Assert.fail("Could not find non-snapshot version of the BOM"); - } - return highestNonsnapshotVersion.get(); - } -} diff --git a/boms/pom.xml b/boms/pom.xml index 507915a380..5aa20037ba 100644 --- a/boms/pom.xml +++ b/boms/pom.xml @@ -29,7 +29,6 @@ - cloud-oss-bom cloud-lts-bom upper-bounds-check integration-tests diff --git a/boms/upper-bounds-check/pom.xml b/boms/upper-bounds-check/pom.xml index 2176315f11..8d9e52a5cc 100644 --- a/boms/upper-bounds-check/pom.xml +++ b/boms/upper-bounds-check/pom.xml @@ -43,7 +43,7 @@ com.google.cloud libraries-bom - 21.0.1-SNAPSHOT + 25.3.1-SNAPSHOT pom import diff --git a/dashboard/README.md b/dashboard/README.md deleted file mode 100644 index 9e83166283..0000000000 --- a/dashboard/README.md +++ /dev/null @@ -1,7 +0,0 @@ -To generate the dashboard from the root directory run: - -``` -$ mvn clean install -$ cd dashboard -$ mvn exec:java -Dexec.arguments="-f ../boms/cloud-oss-bom/pom.xml" -``` diff --git a/dashboard/pom.xml b/dashboard/pom.xml deleted file mode 100644 index b7cdcc9bec..0000000000 --- a/dashboard/pom.xml +++ /dev/null @@ -1,80 +0,0 @@ - - - 4.0.0 - - com.google.cloud.tools - dependencies-parent - 1.5.12-SNAPSHOT - - dashboard - - Cloud Tools Open Source Code Hygiene Dashboard - https://github.com/GoogleCloudPlatform/cloud-opensource-java/dashboard - - Google LLC. - https://www.google.com - - - - - The Apache License, Version 2.0 - http://www.apache.org/licenses/LICENSE-2.0.txt - - - - - UTF-8 - 1.8 - 1.8 - - - - - org.freemarker - freemarker - 2.3.31 - - - com.google.guava - guava - - - ${project.groupId} - dependencies - ${project.version} - - - - xom - xom - 1.3.7 - test - - - junit - junit - test - - - com.google.truth - truth - test - - - - - - - org.codehaus.mojo - exec-maven-plugin - - false - com.google.cloud.tools.opensource.dashboard.DashboardMain - - - - - - diff --git a/dashboard/src/main/java/com/google/cloud/tools/opensource/dashboard/ArtifactCache.java b/dashboard/src/main/java/com/google/cloud/tools/opensource/dashboard/ArtifactCache.java deleted file mode 100644 index ec12246c52..0000000000 --- a/dashboard/src/main/java/com/google/cloud/tools/opensource/dashboard/ArtifactCache.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright 2018 Google LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.cloud.tools.opensource.dashboard; - -import java.util.List; -import java.util.Map; - -import org.eclipse.aether.artifact.Artifact; - -import com.google.cloud.tools.opensource.dependencies.DependencyGraph; - -/** - * Unified return type to bundle a lot of information about multiple artifacts together. - */ -class ArtifactCache { - - private Map infoMap; - private List globalDependencies; - - void setInfoMap(Map infoMap) { - this.infoMap = infoMap; - } - - void setGlobalDependencies(List globalDependencies) { - this.globalDependencies = globalDependencies; - } - - Map getInfoMap() { - return infoMap; - } - - List getGlobalDependencies() { - return globalDependencies; - } - -} diff --git a/dashboard/src/main/java/com/google/cloud/tools/opensource/dashboard/ArtifactInfo.java b/dashboard/src/main/java/com/google/cloud/tools/opensource/dashboard/ArtifactInfo.java deleted file mode 100644 index 1bf711c88e..0000000000 --- a/dashboard/src/main/java/com/google/cloud/tools/opensource/dashboard/ArtifactInfo.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright 2018 Google LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.cloud.tools.opensource.dashboard; - -import org.eclipse.aether.RepositoryException; - -import com.google.cloud.tools.opensource.dependencies.DependencyGraph; - -/** - * Cache of info looked up for an artifact. - */ -class ArtifactInfo { - - private DependencyGraph completeDependencies; - private DependencyGraph transitiveDependencies; - private RepositoryException exception; - - ArtifactInfo(DependencyGraph completeDependencies, - DependencyGraph transitiveDependencies) { - this.completeDependencies = completeDependencies; - this.transitiveDependencies = transitiveDependencies; - } - - ArtifactInfo(RepositoryException ex) { - this.exception = ex; - } - - DependencyGraph getCompleteDependencies() { - return completeDependencies; - } - - DependencyGraph getTransitiveDependencies() { - return transitiveDependencies; - } - - RepositoryException getException() { - return exception; - } - -} diff --git a/dashboard/src/main/java/com/google/cloud/tools/opensource/dashboard/ArtifactResults.java b/dashboard/src/main/java/com/google/cloud/tools/opensource/dashboard/ArtifactResults.java deleted file mode 100644 index da528a2f84..0000000000 --- a/dashboard/src/main/java/com/google/cloud/tools/opensource/dashboard/ArtifactResults.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright 2018 Google LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.cloud.tools.opensource.dashboard; - -import java.util.HashMap; -import java.util.Map; - -import javax.annotation.Nullable; - -import org.eclipse.aether.artifact.Artifact; - -import com.google.cloud.tools.opensource.dependencies.Artifacts; - -/** - * Collection of test results for a single artifact. - */ -public final class ArtifactResults { - - private final Map results = new HashMap<>(); - private final Artifact artifact; - private String exceptionMessage; - - public ArtifactResults(Artifact artifact) { - this.artifact = artifact; - } - - public void setExceptionMessage(String exceptionMessage) { - this.exceptionMessage = exceptionMessage; - } - - void addResult(String testName, int failures) { - results.put(testName, failures); - } - - /** - * @return true for pass, false for fail, null for unknown test - */ - @Nullable - public Boolean getResult(String testName) { - Integer failures = results.get(testName); - if (failures != null) { - return failures == 0; - } - return null; - } - - public String getCoordinates() { - return Artifacts.toCoordinates(artifact); - } - - /** - * @return message of exception occurred when running test, null for no exception - */ - @Nullable - public String getExceptionMessage() { - return exceptionMessage; - } - - /** - * - * @return number of times the specified test failed. Returns 0 - * if the test was not run. - */ - public int getFailureCount(String testName) { - Integer failures = results.get(testName); - if (failures == null) { - return 0; - } - return failures; - } -} diff --git a/dashboard/src/main/java/com/google/cloud/tools/opensource/dashboard/DashboardArguments.java b/dashboard/src/main/java/com/google/cloud/tools/opensource/dashboard/DashboardArguments.java deleted file mode 100644 index 6dda1bb91a..0000000000 --- a/dashboard/src/main/java/com/google/cloud/tools/opensource/dashboard/DashboardArguments.java +++ /dev/null @@ -1,179 +0,0 @@ -/* - * Copyright 2019 Google LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.cloud.tools.opensource.dashboard; - -import com.google.common.collect.ImmutableList; -import java.nio.file.Path; -import java.nio.file.Paths; -import javax.annotation.Nullable; -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.CommandLineParser; -import org.apache.commons.cli.DefaultParser; -import org.apache.commons.cli.HelpFormatter; -import org.apache.commons.cli.Option; -import org.apache.commons.cli.OptionGroup; -import org.apache.commons.cli.Options; -import org.apache.commons.cli.ParseException; - -/** - * Command-line option for {@link DashboardMain}. The tool takes either a pom.xml file path or Maven - * coordinates for a BOM. - */ -final class DashboardArguments { - private static final Options options = configureOptions(); - private static final HelpFormatter helpFormatter = new HelpFormatter(); - - private static final ImmutableList validDependencyMediationValues = - ImmutableList.of("maven", "gradle"); - - private final CommandLine commandLine; - - private DashboardArguments(CommandLine commandLine) { - this.commandLine = commandLine; - } - - /** - * Returns true if the argument for a file is specified. False if the argument for coordinates is - * specified. - * - *

It is guaranteed that either a file path or Maven coordinates for a BOM are available. - */ - boolean hasFile() { - return commandLine.hasOption('f'); - } - - /** - * Returns true if the argument for a versionless coordinates is specified; otherwise false. - * - *

It is guaranteed that either a file path or Maven coordinates for a BOM are available. - */ - boolean hasVersionlessCoordinates() { - return commandLine.hasOption('a'); - } - - /** Returns an absolute path to pom.xml file of a BOM. Null if file is not specified. */ - @Nullable - Path getBomFile() { - if (!commandLine.hasOption('f')) { - return null; - } - // Trim the value so that maven exec plugin can pass arguments with exec.arguments="-f pom.xml" - return Paths.get(commandLine.getOptionValue('f').trim()).toAbsolutePath(); - } - - /** Returns the Maven coordinates of a BOM. Null if coordinates are not specified. */ - @Nullable - String getBomCoordinates() { - if (!commandLine.hasOption('c')) { - return null; - } - return commandLine.getOptionValue('c').trim(); - } - - /** - * Returns the versionless Maven coordinates of a BOM. Null if versionless coordinates are not - * specified. - */ - @Nullable - String getVersionlessCoordinates() { - if (!commandLine.hasOption('a')) { - return null; - } - return commandLine.getOptionValue('a').trim(); - } - - static DashboardArguments readCommandLine(String... arguments) throws ParseException { - CommandLineParser parser = new DefaultParser(); - - try { - // Throws ParseException if required option group ('-f' or '-c') is not specified - CommandLine commandLine = parser.parse(options, arguments); - String dependencyMediationValue = commandLine.getOptionValue('m'); - if (dependencyMediationValue != null - && !validDependencyMediationValues.contains(dependencyMediationValue)) { - throw new ParseException("Valid values for '-m' are " + validDependencyMediationValues); - } - - return new DashboardArguments(commandLine); - } catch (ParseException ex) { - helpFormatter.printHelp("DashboardMain", options); - throw ex; - } - } - - enum DependencyMediationAlgorithm { - MAVEN, - GRADLE, - } - - /** - * Returns dependency mediation algorithm. By default it's {@link - * DependencyMediationAlgorithm#MAVEN}. - */ - DependencyMediationAlgorithm getDependencyMediation() { - if (!commandLine.hasOption('m')) { - return DependencyMediationAlgorithm.MAVEN; - } - String optionValue = commandLine.getOptionValue('m').trim(); - return "maven".equals(optionValue) - ? DependencyMediationAlgorithm.MAVEN - : DependencyMediationAlgorithm.GRADLE; - } - - private static Options configureOptions() { - Options options = new Options(); - OptionGroup inputGroup = new OptionGroup(); - inputGroup.setRequired(true); - - Option inputFileOption = - Option.builder("f").longOpt("bom-file").hasArg().desc("File to a BOM (pom.xml)").build(); - inputGroup.addOption(inputFileOption); - - Option inputCoordinatesOption = - Option.builder("c") - .longOpt("bom-coordinates") - .hasArg() - .desc( - "Maven coordinates of a BOM. For example, com.google.cloud:libraries-bom:1.0.0") - .build(); - inputGroup.addOption(inputCoordinatesOption); - - Option versionlessCoordinatesOption = - Option.builder("a") - .longOpt("all-versions") - .hasArg() - .desc( - "Maven coordinates of a BOM without version. " - + "For example, com.google.cloud:libraries-bom") - .build(); - inputGroup.addOption(versionlessCoordinatesOption); - - Option dependencyMediationOption = - Option.builder("m") - .longOpt("dependency-mediation") - .hasArg() - .desc( - "The dependency mediation algorithm to choose versions. The valid values are:\n" - + "- 'maven' for nearest-win strategy (default)\n" - + "- 'gradle' for highest-win strategy.") - .build(); - options.addOption(dependencyMediationOption); - - options.addOptionGroup(inputGroup); - return options; - } -} diff --git a/dashboard/src/main/java/com/google/cloud/tools/opensource/dashboard/DashboardMain.java b/dashboard/src/main/java/com/google/cloud/tools/opensource/dashboard/DashboardMain.java deleted file mode 100644 index 9b1e1e7f78..0000000000 --- a/dashboard/src/main/java/com/google/cloud/tools/opensource/dashboard/DashboardMain.java +++ /dev/null @@ -1,617 +0,0 @@ -/* - * Copyright 2018 Google LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.cloud.tools.opensource.dashboard; - -import static com.google.common.base.Preconditions.checkArgument; - -import com.google.cloud.tools.opensource.classpath.ClassFile; -import com.google.cloud.tools.opensource.classpath.ClassPathBuilder; -import com.google.cloud.tools.opensource.classpath.ClassPathEntry; -import com.google.cloud.tools.opensource.classpath.ClassPathResult; -import com.google.cloud.tools.opensource.classpath.DependencyMediation; -import com.google.cloud.tools.opensource.classpath.GradleDependencyMediation; -import com.google.cloud.tools.opensource.classpath.LinkageChecker; -import com.google.cloud.tools.opensource.classpath.LinkageProblem; -import com.google.cloud.tools.opensource.dashboard.DashboardArguments.DependencyMediationAlgorithm; -import com.google.cloud.tools.opensource.dependencies.Artifacts; -import com.google.cloud.tools.opensource.dependencies.Bom; -import com.google.cloud.tools.opensource.dependencies.DependencyGraph; -import com.google.cloud.tools.opensource.dependencies.DependencyGraphBuilder; -import com.google.cloud.tools.opensource.dependencies.DependencyPath; -import com.google.cloud.tools.opensource.dependencies.MavenRepositoryException; -import com.google.cloud.tools.opensource.dependencies.RepositoryUtility; -import com.google.cloud.tools.opensource.dependencies.Update; -import com.google.cloud.tools.opensource.dependencies.VersionComparator; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Joiner; -import com.google.common.base.Splitter; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Maps; -import com.google.common.collect.Multimaps; -import com.google.common.collect.Sets; -import freemarker.template.Configuration; -import freemarker.template.DefaultObjectWrapper; -import freemarker.template.DefaultObjectWrapperBuilder; -import freemarker.template.Template; -import freemarker.template.TemplateException; -import freemarker.template.TemplateHashModel; -import freemarker.template.Version; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStreamWriter; -import java.io.Writer; -import java.net.URISyntaxException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.time.LocalDateTime; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.TreeMap; -import org.apache.commons.cli.ParseException; -import org.eclipse.aether.RepositoryException; -import org.eclipse.aether.RepositorySystem; -import org.eclipse.aether.artifact.Artifact; -import org.eclipse.aether.artifact.DefaultArtifact; -import org.eclipse.aether.graph.Dependency; -import org.eclipse.aether.version.InvalidVersionSpecificationException; - -public class DashboardMain { - - public static final String TEST_NAME_LINKAGE_CHECK = "Linkage Errors"; - public static final String TEST_NAME_UPPER_BOUND = "Upper Bounds"; - public static final String TEST_NAME_GLOBAL_UPPER_BOUND = "Global Upper Bounds"; - public static final String TEST_NAME_DEPENDENCY_CONVERGENCE = "Dependency Convergence"; - - private static final Configuration freemarkerConfiguration = configureFreemarker(); - - private static final DependencyGraphBuilder dependencyGraphBuilder = new DependencyGraphBuilder(); - private static final ClassPathBuilder classPathBuilder = - new ClassPathBuilder(dependencyGraphBuilder); - - /** - * Generates a code hygiene dashboard for a BOM. This tool takes a path to pom.xml of the BOM as - * an argument or Maven coordinates to a BOM. - * - *

Generated dashboard is at {@code target/$groupId/$artifactId/$version/index.html}, where - * each value is from BOM coordinates except {@code $version} is "snapshot" if the BOM has - * snapshot version. - */ - public static void main(String[] arguments) - throws IOException, TemplateException, RepositoryException, URISyntaxException, - ParseException, MavenRepositoryException { - DashboardArguments dashboardArguments = DashboardArguments.readCommandLine(arguments); - - if (dashboardArguments.hasVersionlessCoordinates()) { - generateAllVersions( - dashboardArguments.getVersionlessCoordinates(), - dashboardArguments.getDependencyMediation()); - } else if (dashboardArguments.hasFile()) { - generate(dashboardArguments.getBomFile(), dashboardArguments.getDependencyMediation()); - } else { - generate(dashboardArguments.getBomCoordinates(), dashboardArguments.getDependencyMediation()); - } - } - - private static void generateAllVersions( - String versionlessCoordinates, DependencyMediationAlgorithm dependencyMediationAlgorithm) - throws IOException, TemplateException, RepositoryException, URISyntaxException, - MavenRepositoryException { - List elements = Splitter.on(':').splitToList(versionlessCoordinates); - if (elements.size() != 2) { - System.err.println( - "Versionless coordinates should have one colon: " + versionlessCoordinates); - return; - } - String groupId = elements.get(0); - String artifactId = elements.get(1); - - RepositorySystem repositorySystem = RepositoryUtility.newRepositorySystem(); - ImmutableList versions = - RepositoryUtility.findVersions(repositorySystem, groupId, artifactId); - for (String version : versions) { - generate( - String.format("%s:%s:%s", groupId, artifactId, version), dependencyMediationAlgorithm); - } - generateVersionIndex(groupId, artifactId, versions); - } - - @VisibleForTesting - static Path generateVersionIndex(String groupId, String artifactId, List versions) - throws IOException, TemplateException, URISyntaxException { - Path directory = outputDirectory(groupId, artifactId, "snapshot").getParent(); - directory.toFile().mkdirs(); - Path page = directory.resolve("index.html"); - - Map templateData = new HashMap<>(); - templateData.put("versions", versions); - templateData.put("groupId", groupId); - templateData.put("artifactId", artifactId); - - File dashboardFile = page.toFile(); - try (Writer out = - new OutputStreamWriter(new FileOutputStream(dashboardFile), StandardCharsets.UTF_8)) { - Template dashboard = freemarkerConfiguration.getTemplate("/templates/version_index.ftl"); - dashboard.process(templateData, out); - } - - copyResource(directory, "css/dashboard.css"); - - return page; - } - - @VisibleForTesting - static Path generate( - String bomCoordinates, DependencyMediationAlgorithm dependencyMediationAlgorithm) - throws IOException, TemplateException, RepositoryException, URISyntaxException { - Path output = generate(Bom.readBom(bomCoordinates), dependencyMediationAlgorithm); - System.out.println("Wrote dashboard for " + bomCoordinates + " to " + output); - return output; - } - - @VisibleForTesting - static Path generate(Path bomFile, DependencyMediationAlgorithm dependencyMediationAlgorithm) - throws IOException, TemplateException, URISyntaxException, MavenRepositoryException, - InvalidVersionSpecificationException { - checkArgument(Files.isRegularFile(bomFile), "The input BOM %s is not a regular file", bomFile); - checkArgument(Files.isReadable(bomFile), "The input BOM %s is not readable", bomFile); - Path output = generate(Bom.readBom(bomFile), dependencyMediationAlgorithm); - System.out.println("Wrote dashboard for " + bomFile + " to " + output); - return output; - } - - private static Path generate(Bom bom, DependencyMediationAlgorithm dependencyMediationAlgorithm) - throws IOException, TemplateException, URISyntaxException, - InvalidVersionSpecificationException { - - ImmutableList managedDependencies = bom.getManagedDependencies(); - - DependencyMediation dependencyMediation = - dependencyMediationAlgorithm == DependencyMediationAlgorithm.MAVEN - ? DependencyMediation.MAVEN - : GradleDependencyMediation.withEnforcedPlatform(bom); - - ClassPathResult classPathResult = - classPathBuilder.resolve(managedDependencies, false, dependencyMediation); - ImmutableList classpath = classPathResult.getClassPath(); - - LinkageChecker linkageChecker = LinkageChecker.create(classpath); - - ImmutableSet linkageProblems = linkageChecker.findLinkageProblems(); - - ArtifactCache cache = loadArtifactInfo(managedDependencies); - Path output = generateHtml(bom, cache, classPathResult, linkageProblems); - - return output; - } - - private static Path outputDirectory(String groupId, String artifactId, String version) { - String versionPathElement = version.contains("-SNAPSHOT") ? "snapshot" : version; - return Paths.get("target", groupId, artifactId, versionPathElement); - } - - private static Path generateHtml( - Bom bom, - ArtifactCache cache, - ClassPathResult classPathResult, - ImmutableSet linkageProblems) - throws IOException, TemplateException, URISyntaxException { - - Artifact bomArtifact = new DefaultArtifact(bom.getCoordinates()); - - Path relativePath = - outputDirectory( - bomArtifact.getGroupId(), bomArtifact.getArtifactId(), bomArtifact.getVersion()); - Path output = Files.createDirectories(relativePath); - - copyResource(output, "css/dashboard.css"); - copyResource(output, "js/dashboard.js"); - - ImmutableMap> linkageProblemTable = - indexByJar(linkageProblems); - - List table = - generateReports( - freemarkerConfiguration, output, cache, linkageProblemTable, classPathResult, bom); - - generateDashboard( - freemarkerConfiguration, - output, - table, - cache.getGlobalDependencies(), - linkageProblemTable, - classPathResult, - bom); - - return output; - } - - private static void copyResource(Path output, String resourceName) - throws IOException, URISyntaxException { - ClassLoader classLoader = DashboardMain.class.getClassLoader(); - Path input = Paths.get(classLoader.getResource(resourceName).toURI()).toAbsolutePath(); - Path copy = output.resolve(input.getFileName()); - if (!Files.exists(copy)) { - Files.copy(input, copy); - } - } - - @VisibleForTesting - static Configuration configureFreemarker() { - Configuration configuration = new Configuration(new Version("2.3.28")); - configuration.setDefaultEncoding("UTF-8"); - configuration.setClassForTemplateLoading(DashboardMain.class, "/"); - return configuration; - } - - @VisibleForTesting - static List generateReports( - Configuration configuration, - Path output, - ArtifactCache cache, - ImmutableMap> linkageProblemTable, - ClassPathResult classPathResult, - Bom bom) - throws TemplateException { - - Map artifacts = cache.getInfoMap(); - List table = new ArrayList<>(); - for (Entry entry : artifacts.entrySet()) { - ArtifactInfo info = entry.getValue(); - try { - if (info.getException() != null) { - ArtifactResults unavailable = new ArtifactResults(entry.getKey()); - unavailable.setExceptionMessage(info.getException().getMessage()); - table.add(unavailable); - } else { - Artifact artifact = entry.getKey(); - ImmutableSet jarsInDependencyTree = - classPathResult.getClassPathEntries(Artifacts.toCoordinates(artifact)); - Map> relevantLinkageProblemTable = - Maps.filterKeys(linkageProblemTable, jarsInDependencyTree::contains); - - ArtifactResults results = - generateArtifactReport( - configuration, - output, - artifact, - entry.getValue(), - cache.getGlobalDependencies(), - ImmutableMap.copyOf(relevantLinkageProblemTable), - classPathResult, - bom); - table.add(results); - } - } catch (IOException ex) { - ArtifactResults unavailableTestResult = new ArtifactResults(entry.getKey()); - unavailableTestResult.setExceptionMessage(ex.getMessage()); - // Even when there's a problem generating test result, show the error in the dashboard - table.add(unavailableTestResult); - } - } - - return table; - } - - /** - * This is the only method that queries the Maven repository. - */ - private static ArtifactCache loadArtifactInfo(List artifacts) { - Map infoMap = new LinkedHashMap<>(); - List globalDependencies = new ArrayList<>(); - - for (Artifact artifact : artifacts) { - DependencyGraph completeDependencies = - dependencyGraphBuilder.buildVerboseDependencyGraph(artifact); - globalDependencies.add(completeDependencies); - - // picks versions according to Maven rules - DependencyGraph transitiveDependencies = - dependencyGraphBuilder.buildMavenDependencyGraph(new Dependency(artifact, "compile")); - - ArtifactInfo info = new ArtifactInfo(completeDependencies, transitiveDependencies); - infoMap.put(artifact, info); - } - - ArtifactCache cache = new ArtifactCache(); - cache.setInfoMap(infoMap); - cache.setGlobalDependencies(globalDependencies); - - return cache; - } - - private static ArtifactResults generateArtifactReport( - Configuration configuration, - Path output, - Artifact artifact, - ArtifactInfo artifactInfo, - List globalDependencies, - ImmutableMap> linkageProblemTable, - ClassPathResult classPathResult, - Bom bom) - throws IOException, TemplateException { - - String coordinates = Artifacts.toCoordinates(artifact); - File outputFile = output.resolve(coordinates.replace(':', '_') + ".html").toFile(); - - try (Writer out = new OutputStreamWriter( - new FileOutputStream(outputFile), StandardCharsets.UTF_8)) { - - // includes all versions - DependencyGraph graph = artifactInfo.getCompleteDependencies(); - List convergenceIssues = graph.findUpdates(); - - // picks versions according to Maven rules - DependencyGraph transitiveDependencies = artifactInfo.getTransitiveDependencies(); - - Map upperBoundFailures = - findUpperBoundsFailures(graph.getHighestVersionMap(), transitiveDependencies); - - Map globalUpperBoundFailures = findUpperBoundsFailures( - collectLatestVersions(globalDependencies), transitiveDependencies); - - long totalLinkageErrorCount = - linkageProblemTable.values().stream() - .flatMap(problemToClasses -> problemToClasses.stream().map(LinkageProblem::getSymbol)) - .distinct() // The dashboard counts linkage errors by the symbols - .count(); - - Template report = configuration.getTemplate("/templates/component.ftl"); - - Map templateData = new HashMap<>(); - - DefaultObjectWrapper wrapper = - new DefaultObjectWrapperBuilder(Configuration.VERSION_2_3_28).build(); - TemplateHashModel staticModels = wrapper.getStaticModels(); - templateData.put("linkageProblem", staticModels.get(LinkageProblem.class.getName())); - - templateData.put("artifact", artifact); - templateData.put("updates", convergenceIssues); - templateData.put("upperBoundFailures", upperBoundFailures); - templateData.put("globalUpperBoundFailures", globalUpperBoundFailures); - templateData.put("lastUpdated", LocalDateTime.now()); - templateData.put("dependencyGraph", graph); - templateData.put("linkageProblems", linkageProblemTable); - templateData.put("classPathResult", classPathResult); - templateData.put("totalLinkageErrorCount", totalLinkageErrorCount); - templateData.put("coordinates", bom.getCoordinates()); - - report.process(templateData, out); - - ArtifactResults results = new ArtifactResults(artifact); - results.addResult(TEST_NAME_UPPER_BOUND, upperBoundFailures.size()); - results.addResult(TEST_NAME_GLOBAL_UPPER_BOUND, globalUpperBoundFailures.size()); - results.addResult(TEST_NAME_DEPENDENCY_CONVERGENCE, convergenceIssues.size()); - results.addResult(TEST_NAME_LINKAGE_CHECK, (int) totalLinkageErrorCount); - - return results; - } - } - - private static Map findUpperBoundsFailures( - Map expectedVersionMap, - DependencyGraph transitiveDependencies) { - - Map actualVersionMap = transitiveDependencies.getHighestVersionMap(); - - VersionComparator comparator = new VersionComparator(); - - Map upperBoundFailures = new LinkedHashMap<>(); - - for (String id : expectedVersionMap.keySet()) { - String expectedVersion = expectedVersionMap.get(id); - String actualVersion = actualVersionMap.get(id); - // Check that the actual version is not null because it is - // possible for dependencies to appear or disappear from the tree - // depending on which version of another dependency is loaded. - // In both cases, no action is needed. - if (actualVersion != null && comparator.compare(actualVersion, expectedVersion) < 0) { - // Maven did not choose highest version - DefaultArtifact lower = new DefaultArtifact(id + ":" + actualVersion); - DefaultArtifact upper = new DefaultArtifact(id + ":" + expectedVersion); - upperBoundFailures.put(lower, upper); - } - } - return upperBoundFailures; - } - - /** - * Partitions {@code linkageProblems} by the JAR file that contains the {@link ClassFile}. - * - *

For example, {@code classes = result.get(JarX).get(linkageProblemY)} where {@code classes} - * are not null means that {@code JarX} has {@code linkageProblemY} and that {@code JarX} contains - * {@code classes} which reference {@code linkageProblemY.getSymbol()}. - */ - private static ImmutableMap> indexByJar( - ImmutableSet linkageProblems) { - - ImmutableMap> jarMap = - Multimaps.index(linkageProblems, problem -> problem.getSourceClass().getClassPathEntry()) - .asMap(); - - return ImmutableMap.copyOf(Maps.transformValues(jarMap, ImmutableSet::copyOf)); - } - - @VisibleForTesting - static void generateDashboard( - Configuration configuration, - Path output, - List table, - List globalDependencies, - ImmutableMap> linkageProblemTable, - ClassPathResult classPathResult, - Bom bom) - throws IOException, TemplateException { - - Map latestArtifacts = collectLatestVersions(globalDependencies); - - Map templateData = new HashMap<>(); - templateData.put("table", table); - templateData.put("lastUpdated", LocalDateTime.now()); - templateData.put("latestArtifacts", latestArtifacts); - templateData.put("linkageProblems", linkageProblemTable); - templateData.put("classPathResult", classPathResult); - templateData.put("dependencyPathRootCauses", findRootCauses(classPathResult)); - templateData.put("coordinates", bom.getCoordinates()); - templateData.put("dependencyGraphs", globalDependencies); - - // Accessing static methods from Freemarker template - // https://freemarker.apache.org/docs/pgui_misc_beanwrapper.html#autoid_60 - DefaultObjectWrapper wrapper = new DefaultObjectWrapperBuilder(Configuration.VERSION_2_3_28) - .build(); - TemplateHashModel staticModels = wrapper.getStaticModels(); - templateData.put("dashboardMain", staticModels.get(DashboardMain.class.getName())); - templateData.put("pieChart", staticModels.get(PieChart.class.getName())); - templateData.put("linkageProblem", staticModels.get(LinkageProblem.class.getName())); - - File dashboardFile = output.resolve("index.html").toFile(); - try (Writer out = new OutputStreamWriter( - new FileOutputStream(dashboardFile), StandardCharsets.UTF_8)) { - Template dashboard = configuration.getTemplate("/templates/index.ftl"); - dashboard.process(templateData, out); - } - - File detailsFile = output.resolve("artifact_details.html").toFile(); - try (Writer out = new OutputStreamWriter( - new FileOutputStream(detailsFile), StandardCharsets.UTF_8)) { - Template details = configuration.getTemplate("/templates/artifact_details.ftl"); - details.process(templateData, out); - } - - File unstable = output.resolve("unstable_artifacts.html").toFile(); - try (Writer out = new OutputStreamWriter( - new FileOutputStream(unstable), StandardCharsets.UTF_8)) { - Template details = configuration.getTemplate("/templates/unstable_artifacts.ftl"); - details.process(templateData, out); - } - - File dependencyTrees = output.resolve("dependency_trees.html").toFile(); - try (Writer out = - new OutputStreamWriter(new FileOutputStream(dependencyTrees), StandardCharsets.UTF_8)) { - Template details = configuration.getTemplate("/templates/dependency_trees.ftl"); - details.process(templateData, out); - } - } - - private static Map collectLatestVersions( - List globalDependencies) { - Map latestArtifacts = new TreeMap<>(); - VersionComparator comparator = new VersionComparator(); - - if (globalDependencies != null) { - for (DependencyGraph graph : globalDependencies) { - Map map = graph.getHighestVersionMap(); - for (String key : map.keySet()) { - String newVersion = map.get(key); - String oldVersion = latestArtifacts.get(key); - if (oldVersion == null || comparator.compare(newVersion, oldVersion) > 0) { - latestArtifacts.put(key, map.get(key)); - } - } - } - } - return latestArtifacts; - } - - /** - * Returns the number of rows in {@code table} that show unavailable ({@code null} result) or some - * failures for {@code columnName}. - */ - public static long countFailures(List table, String columnName) { - return table.stream() - .filter(row -> row.getResult(columnName) == null || row.getFailureCount(columnName) > 0) - .count(); - } - - private static final int MINIMUM_NUMBER_DEPENDENCY_PATHS = 5; - - /** - * Returns mapping from jar files to summaries of the root problem in their {@link - * DependencyPath}s. The summary explains common patterns ({@code groupId:artifactId}) in the path - * elements. The returned map does not have a key for a jar file when it has fewer than {@link - * #MINIMUM_NUMBER_DEPENDENCY_PATHS} dependency paths or a common pattern is not found among the - * elements in the paths. - * - *

Example summary: "Artifacts 'com.google.http-client:google-http-client > - * commons-logging:commons-logging > log4j:log4j' exist in all 994 dependency paths. Example - * path: com.google.cloud:google-cloud-core:1.59.0 ..." - * - *

Using this summary in the BOM dashboard avoids repetitive items in the {@link - * DependencyPath} list that share the same root problem caused by widely-used libraries, for - * example, {@code commons-logging:commons-logging}, {@code - * com.google.http-client:google-http-client} and {@code log4j:log4j}. - */ - private static ImmutableMap findRootCauses(ClassPathResult classPathResult) { - // Freemarker is not good at handling non-string keys. Path object in .ftl is automatically - // converted to String. https://freemarker.apache.org/docs/app_faq.html#faq_nonstring_keys - ImmutableMap.Builder builder = ImmutableMap.builder(); - - for (ClassPathEntry entry : classPathResult.getClassPath()) { - List dependencyPaths = classPathResult.getDependencyPaths(entry); - - ImmutableList commonVersionlessArtifacts = - commonVersionlessArtifacts(dependencyPaths); - - if (dependencyPaths.size() > MINIMUM_NUMBER_DEPENDENCY_PATHS - && commonVersionlessArtifacts.size() > 1) { // The last paths elements are always same - builder.put( - entry.toString(), - summaryMessage( - dependencyPaths.size(), commonVersionlessArtifacts, dependencyPaths.get(0))); - } - } - return builder.build(); - } - - private static ImmutableList commonVersionlessArtifacts( - List dependencyPaths) { - ImmutableList initialVersionlessCoordinates = - dependencyPaths.get(0).getArtifactKeys(); - // LinkedHashSet remembers insertion order - LinkedHashSet versionlessCoordinatesIntersection = - Sets.newLinkedHashSet(initialVersionlessCoordinates); - for (DependencyPath dependencyPath : dependencyPaths) { - // List of versionless coordinates ("groupId:artifactId") - ImmutableList versionlessCoordinatesInPath = dependencyPath.getArtifactKeys(); - // intersection of elements in DependencyPaths - versionlessCoordinatesIntersection.retainAll(versionlessCoordinatesInPath); - } - - return ImmutableList.copyOf(versionlessCoordinatesIntersection); - } - - private static String summaryMessage( - int dependencyPathCount, List coordinates, DependencyPath examplePath) { - StringBuilder messageBuilder = new StringBuilder(); - messageBuilder.append("Dependency path '"); - messageBuilder.append(Joiner.on(" > ").join(coordinates)); - messageBuilder.append("' exists in all " + dependencyPathCount + " dependency paths. "); - messageBuilder.append("Example path: "); - messageBuilder.append(examplePath); - return messageBuilder.toString(); - } -} diff --git a/dashboard/src/main/java/com/google/cloud/tools/opensource/dashboard/PieChart.java b/dashboard/src/main/java/com/google/cloud/tools/opensource/dashboard/PieChart.java deleted file mode 100644 index 8b4ecd75f7..0000000000 --- a/dashboard/src/main/java/com/google/cloud/tools/opensource/dashboard/PieChart.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright 2019 Google LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.cloud.tools.opensource.dashboard; - -import java.awt.geom.Point2D; - -public class PieChart { - - /** - * Calculate SVG arc end for a pie piece. Assumes the piece starts at the top of the circle. - * - * Do not forget: - * - * 1. SVG origin starts at top left. - * 2. x increases to the right and y increases **down**. - */ - static Point2D calculateEndPoint(double radius, double centerX, double centerY, double ratio) { - if (ratio > 1) { - ratio = 1.0; - } - - double radians = ratio * 2 * Math.PI; - - // Since we're starting at the top of the circle this is rotated 90 degrees - // from the normal coordinates. This is why we use sine for x and cosine for y. - double x = radius * (1 + Math.sin(radians)); - double y = radius * (1 - Math.cos(radians)); - return new Point2D.Double(x + centerX - radius, y + centerY - radius); - } - - // so I can avoid teaching FreeMarker how to wrap a java.awt.Point - public static double calculateEndPointX( - double radius, double centerX, double centerY, double ratio) { - return calculateEndPoint(radius, centerX, centerY, ratio).getX(); - } - - public static double calculateEndPointY( - double radius, double centerX, double centerY, double ratio) { - return calculateEndPoint(radius, centerX, centerY, ratio).getY(); - } -} diff --git a/dashboard/src/main/resources/css/dashboard.css b/dashboard/src/main/resources/css/dashboard.css deleted file mode 100644 index 9997f125cc..0000000000 --- a/dashboard/src/main/resources/css/dashboard.css +++ /dev/null @@ -1,147 +0,0 @@ -body { - font-family: "Poppins", sans-serif; - font-weight: 400; - line-height: 1.625; - margin-left: 2em; -} - -h1, -h2, -h3 { - color: #333333; - font-weight: 700; - margin: 0; - line-height: 1.2; - margin-top: 1ex; - margin-bottom: 1ex; -} - -h1 { - font-size: 3em; -} - -h2 { - font-size: 2.5em; -} - -h3 { - font-size: 2em; -} - -th, td { - padding: 5pt; -} - -pre { - line-height: normal; -} - -.pass { - background-color: lightgreen; - font-weight: bold; -} - -.fail { - background-color: pink; - font-weight: bold; -} - -.unavailable { - background-color: gray; - font-weight: bold; -} - -p.dependency-tree-node { - margin-top: 0; - margin-bottom: 0; -} - -.linkage-check-dependency-paths, .jar-linkage-report { - margin-left: 1em; -} - -p.jar-linkage-report-cause { - margin-bottom: 0; - margin-left: 2em; -} - -ul.jar-linkage-report-cause { - margin-top: 0; - margin-left: 3em; -} - -ul.jar-linkage-report-cause > li { - font: 1em 'Droid Sans Mono', monospace; -} - - /* ----- Statistic ----- */ -.statistics { - padding-top: 50px; -} - -.container { - min-height: 20ex; - display: flex; - flex-wrap: wrap; - align-items: flex-start; -} - -.statistic-item { - flex: 0 0 16em; - padding: 1.6em 2.5em; - position: relative; - min-height: 180px; - overflow: hidden; - margin-bottom: 40px; - border: none; - border-radius: 3px; - box-shadow: 0px 10px 20px 0px rgba(0, 0, 0, 0.03); - margin: 1em; - margin-left: unset; - margin-right: 2em; -} - -.statistic-item .desc { - font-size: 1.15em; - text-transform: uppercase; - font-weight: 300; - color: rgba(255, 255, 255, 0.6); -} - -.statistic-item h2 { - font-size: 2.3em; - font-weight: 300; - color: #fff; -} - -.statistic-item-green { - background: #00b26f; -} - -.statistic-item-orange { - background: #ff8300; -} - -.statistic-item-blue { - background: #00b5e9; -} - -.statistic-item-red { - background: #fa4251; -} - -.statistic-item-yellow { - background: #f1c40f; -} - -#piecharts th { - vertical-align: top; -} - -#piecharts td { - vertical-align: top; -} - -.pie { - "text-align: center" -} diff --git a/dashboard/src/main/resources/js/dashboard.js b/dashboard/src/main/resources/js/dashboard.js deleted file mode 100644 index 2a2128c45a..0000000000 --- a/dashboard/src/main/resources/js/dashboard.js +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright 2019 Google LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Toggles the visibility of an HTML element below the button. - * @param button clicked button element - */ -function toggleNextSiblingVisibility(button) { - const nextSibling = button.parentElement.nextElementSibling; - const currentVisibility = nextSibling.style.display !== "none"; - const nextVisibility = !currentVisibility; - nextSibling.style.display = nextVisibility ? "" : "none"; - button.innerText = nextVisibility ? "▼" : "▶"; -} diff --git a/dashboard/src/main/resources/poms/demo.xml b/dashboard/src/main/resources/poms/demo.xml deleted file mode 100644 index 7167c0415a..0000000000 --- a/dashboard/src/main/resources/poms/demo.xml +++ /dev/null @@ -1,49 +0,0 @@ - - 4.0.0 - - com.google.cloud - demo - 1.0-SNAPSHOT - jar - - demo-pom - - - UTF-8 - 1.8 - - - - - ${dependencyGroupId} - ${dependencyArtifactId} - ${dependencyVersion} - test - - - - - - - org.apache.maven.plugins - maven-enforcer-plugin - 3.0.0-M1 - - - enforce - - enforce - - - - - - - - - - - - \ No newline at end of file diff --git a/dashboard/src/main/resources/templates/artifact_details.ftl b/dashboard/src/main/resources/templates/artifact_details.ftl deleted file mode 100644 index 75acdcd51b..0000000000 --- a/dashboard/src/main/resources/templates/artifact_details.ftl +++ /dev/null @@ -1,55 +0,0 @@ - - - <#include "macros.ftl"> - - - Google Cloud Platform Java Open Source Dependency Dashboard Artifact Details Table - - - - -

Google Cloud Platform Java Dependency Dashboard Artifact Details

-

BOM: ${coordinates?html}

-

Dependency Details

- - - - - - - - - - <#list table as row> - <#assign report_url = row.getCoordinates()?replace(":", "_") + '.html' /> - - - <#-- The name key should match TEST_NAME_XXXX variables --> - <@testResult row=row name="Linkage Errors"/> - <@testResult row=row name="Upper Bounds"/> - <@testResult row=row name="Global Upper Bounds"/> - <@testResult row=row name="Dependency Convergence"/> - - -
Artifact - Linkage Check - Upper Bounds - Global Upper Bounds - Dependency Convergence
${row.getCoordinates()}
- -
- -

Linkage Errors

- - <#list linkageProblems as jar, problems> - <@formatJarLinkageReport jar problems classPathResult dependencyPathRootCauses/> - - -
-

Last generated at ${lastUpdated}

- - \ No newline at end of file diff --git a/dashboard/src/main/resources/templates/component.ftl b/dashboard/src/main/resources/templates/component.ftl deleted file mode 100644 index afdc26cc09..0000000000 --- a/dashboard/src/main/resources/templates/component.ftl +++ /dev/null @@ -1,156 +0,0 @@ - - - <#include "macros.ftl"> - <#assign groupId = artifact.getGroupId()> - <#assign artifactId = artifact.getArtifactId()> - <#assign version = artifact.getVersion()> - - - Google Cloud Platform Dependency Analysis Report for ${groupId}:${artifactId}:${version} - - - - - -

Dependency Analysis of ${groupId}:${artifactId}:${version}

-

BOM: ${coordinates?html}

-

Global Upper Bounds Check

- -

For each transitive dependency the library pulls in, the highest version - found anywhere in the union of the BOM's dependency trees is picked.

- - <#if globalUpperBoundFailures?size gt 0> -

Global Upper Bounds Fixes

- -

Suggested updates to bring this artifact into sync with the highest versions - of its dependencies used by any artifact in the BOM:

- -
    - <#list globalUpperBoundFailures as lower, upper> - <#if lower.getGroupId() == groupId && lower.getArtifactId() == artifactId - && lower.getVersion() == version ><#-- Not checking 'file' attribute of Artifact --> - <#-- When this is upgrading a BOM member --> -
  • - Upgrade ${lower} in the BOM to version "${upper.getVersion()}": - -

    Update the version of managed dependency element - ${groupId}:${artifactId} in the BOM:

    - - 
    <dependencyManagement>
-  <dependencies>
-    ...
-    <dependency>
-      <groupId>${upper.getGroupId()}</groupId>
-      <artifactId>${upper.getArtifactId()}</artifactId>
-      <version>${upper.getVersion()}</version>
-    </dependency>
    - -
    • - <#else > -
  • - Upgrade ${lower} to version "${upper.getVersion()}": - -

    Add this dependency element to the pom.xml for ${groupId}:${artifactId}:${version}: -

    - - 
    <dependency>
-<groupId>${upper.getGroupId()}</groupId>
-<artifactId>${upper.getArtifactId()}</artifactId>
-<version>${upper.getVersion()}</version>
-</dependency>
    - -

    If the pom.xml for ${groupId}:${artifactId}:${version} already includes this - dependency, update the version of the existing dependency element. - Otherwise add a new dependency element to the - dependencyManagement section.

    -
    • - - -
- - <#else> -

- ${groupId}:${artifactId}:${version} selects the highest version of all dependencies. -

- - - -

Local Upper Bounds Check

- - -

For each transitive dependency the library pulls in, the highest version - found anywhere in the dependency tree is picked.

- - <#if upperBoundFailures?size gt 0> -

Upper Bounds Fixes

- -

Suggested updates to bring this artifact into sync with the highest versions - of each dependency found in its own dependency tree:

- -
    - <#list upperBoundFailures as lower, upper> -
  • Upgrade ${lower} to ${upper}: - -

    Add this dependency element to the pom.xml for ${groupId}:${artifactId}:${version}:

    - -
    <dependency>
-  <groupId>${upper.getGroupId()}</groupId>
-  <artifactId>${upper.getArtifactId()}</artifactId>
-  <version>${upper.getVersion()}</version>
-</dependency>
    - -
    • - -
- - <#else> -

- ${groupId}:${artifactId}:${version} selects the highest version of all dependencies. -

- - -

Dependency Convergence

- -

There is exactly one version of each dependency in the library's transitive dependency tree. - That is, two artifacts with the same group ID and artifact ID but different versions - do not appear in the tree. No dependency mediation is necessary.

- - <#if updates?size gt 0> -

Suggested Dependency Updates

- -

Caution: The algorithm for suggesting updates is imperfect. - They are not ordered by importance, and one change - may render another moot.

- -

Suggested updates to bring this artifact and its dependencies - into sync with the highest versions - of each dependency found in its own dependency tree:

- -
    - <#list updates as update> -
  • ${update}
    • - -
- <#else> -

${groupId}:${artifactId}:${version} Converges

- - - -

Linkage Check

- -

${totalLinkageErrorCount} linkage error(s)

- <#list linkageProblems as jar, problems> - <@formatJarLinkageReport jar problems classPathResult {} /> - - -

Dependencies

- - <#if dependencyGraph?? > - <@formatDependencyGraph dependencyGraph dependencyGraph.getRootPath() "" /> - <#else> -

Dependency information is unavailable

- -
-

Last generated at ${lastUpdated}

- - \ No newline at end of file diff --git a/dashboard/src/main/resources/templates/dependency_trees.ftl b/dashboard/src/main/resources/templates/dependency_trees.ftl deleted file mode 100644 index 6641141c2c..0000000000 --- a/dashboard/src/main/resources/templates/dependency_trees.ftl +++ /dev/null @@ -1,22 +0,0 @@ - - - <#include "macros.ftl"> - - - Google Cloud Platform Java Open Source Dependency Dashboard: Dependency Trees - - - - -

Dependency Tree of the Artifacts in ${coordinates}

-

BOM: ${coordinates?html}

- <#list dependencyGraphs as graph> - <#assign rootPath = graph.getRootPath() /> -

Dependency Tree of ${rootPath.getLeaf()?html}

- <@formatDependencyGraph graph rootPath "" /> - - -
-

Last generated at ${lastUpdated}

- - \ No newline at end of file diff --git a/dashboard/src/main/resources/templates/index.ftl b/dashboard/src/main/resources/templates/index.ftl deleted file mode 100644 index 36d5d1dade..0000000000 --- a/dashboard/src/main/resources/templates/index.ftl +++ /dev/null @@ -1,142 +0,0 @@ - - - <#include "macros.ftl"> - - - Google Cloud Platform Java Open Source Dependency Dashboard - - - - -

${coordinates} Dependency Status

-
- <#assign totalArtifacts = table?size> - -
-
-
-

${table?size}

- Total Artifacts Checked -
-
- <#assign linkageErrorCount = dashboardMain.countFailures(table, "Linkage Errors")> -

${linkageErrorCount}

- ${(linkageErrorCount == 1)?then("Has", "Have")} Linkage Errors -
-
- <#assign localUpperBoundsErrorCount = dashboardMain.countFailures(table, "Upper Bounds")> -

${dashboardMain.countFailures(table, "Upper Bounds")}

- ${(localUpperBoundsErrorCount == 1)?then("Has", "Have")} Upper Bounds Errors -
-
- <#assign globalUpperBoundsErrorCount = dashboardMain.countFailures(table, "Global Upper Bounds")> -

${dashboardMain.countFailures(table, "Global Upper Bounds")}

- ${(globalUpperBoundsErrorCount == 1)?then("Has", "Have")} Global Upper Bounds Errors -
-
- <#assign convergenceErrorCount = dashboardMain.countFailures(table, "Dependency Convergence")> -

${dashboardMain.countFailures(table, "Dependency Convergence")}

- ${(convergenceErrorCount == 1)?then("Fails", "Fail")} to Converge -
-
-
- - <#assign pieSize = 300 > -
- - - - - - - - - - - - - - - - - - - - - - - -
- Linkage Errors - Local Upper Bounds - Global Upper Bounds - Dependency Convergence
${linkageErrorCount} out of ${totalArtifacts} artifacts - ${plural(linkageErrorCount, "has", "have")} linkage errors. - ${localUpperBoundsErrorCount} out of ${totalArtifacts} artifacts - ${plural(localUpperBoundsErrorCount, "does not", "do not")} pick the - latest versions of all artifacts in their own dependency tree. - ${globalUpperBoundsErrorCount} out of ${totalArtifacts} artifacts - ${plural(globalUpperBoundsErrorCount, "does not", "do not")} select the - most recent version of all artifacts in the BOM.${convergenceErrorCount} out of ${totalArtifacts} artifacts - ${plural(convergenceErrorCount, "fails", "fail")} to converge. -
- <@pieChartSvg - description="${linkageErrorCount} out of ${totalArtifacts} artifacts have linkage - errors." - ratio=linkageErrorCount / totalArtifacts /> - - <#assign doesNot=plural(localUpperBoundsErrorCount, "does not", "do not")> - <@pieChartSvg - description="${localUpperBoundsErrorCount} out of ${totalArtifacts} artifacts - $doesNot pick the - latest versions of all artifacts in their own dependency tree." - ratio=localUpperBoundsErrorCount / totalArtifacts /> - - <@pieChartSvg - description="${globalUpperBoundsErrorCount} out of ${totalArtifacts} artifacts have - global upper bounds errors." - ratio=globalUpperBoundsErrorCount / totalArtifacts /> - - <#assign fails=plural(convergenceErrorCount, "fails", "fail")/> - <@pieChartSvg - description="${convergenceErrorCount} out of ${totalArtifacts} artifacts - ${fails} to converge." - ratio=convergenceErrorCount / totalArtifacts /> -
-
- -

- BOM source code -

- -

- Detailed Artifact Reports -

- -

- Pre 1.0 Artifacts -

- -

Recommended Versions

- -

These are the most recent versions of dependencies used by any of the covered artifacts.

- - - -

- Dependency Trees -

- -
- -

Last generated at ${lastUpdated}

- - \ No newline at end of file diff --git a/dashboard/src/main/resources/templates/macros.ftl b/dashboard/src/main/resources/templates/macros.ftl deleted file mode 100644 index 3d160cdcee..0000000000 --- a/dashboard/src/main/resources/templates/macros.ftl +++ /dev/null @@ -1,142 +0,0 @@ -<#function pluralize number singularNoun pluralNoun> - <#local plural = number gt 1 /> - <#return number + " " + plural?string(pluralNoun, singularNoun)> - -<#-- same as above but without the number --> -<#function plural number singularNoun pluralNoun> - <#local plural = number gt 1 /> - <#return plural?string(pluralNoun, singularNoun)> - - -<#macro formatJarLinkageReport classPathEntry linkageProblems classPathResult - dependencyPathRootCauses> - <#-- problemsToClasses: ImmutableMap> to get key and set of - values in Freemarker --> - <#assign problemsToClasses = linkageProblem.groupBySymbolProblem(linkageProblems) /> - <#assign symbolProblemCount = problemsToClasses?size /> - <#assign referenceCount = 0 /> - <#list problemsToClasses?values as classes> - <#assign referenceCount += classes?size /> - - -

${classPathEntry?html}

-

- ${pluralize(symbolProblemCount, "target class", "target classes")} - causing linkage errors referenced from - ${pluralize(referenceCount, "source class", "source classes")}. -

- <#list problemsToClasses as problem, sourceClasses> - <#if sourceClasses?size == 1> - <#assign sourceClass = sourceClasses[0] /> -

${problem?html}, referenced from ${sourceClass?html}

- <#else> -

${problem?html}, referenced from ${ - pluralize(sourceClasses?size, "class", "classes")?html} - -

- - - - - <#assign jarsInProblem = {} > - <#list linkageProblems as problem> - <#if (problem.getTargetClass())?? > - <#assign targetClassPathEntry = problem.getTargetClass().getClassPathEntry() /> - <#-- Freemarker's hash requires its keys to be strings. - https://freemarker.apache.org/docs/app_faq.html#faq_nonstring_keys --> - <#assign jarsInProblem = jarsInProblem + { targetClassPathEntry.toString() : targetClassPathEntry } > - - - <#list jarsInProblem?values as jarInProblem> - <@showDependencyPath dependencyPathRootCauses classPathResult jarInProblem /> - - <#if !jarsInProblem?values?seq_contains(classPathEntry) > - <@showDependencyPath dependencyPathRootCauses classPathResult classPathEntry /> - - - - -<#macro showDependencyPath dependencyPathRootCauses classPathResult classPathEntry> - <#assign dependencyPaths = classPathResult.getDependencyPaths(classPathEntry) /> - <#assign hasRootCause = dependencyPathRootCauses[classPathEntry]?? /> - <#assign hideDependencyPathsByDefault = (!hasRootCause) && (dependencyPaths?size > 5) /> -

- The following ${plural(dependencyPaths?size, "path contains", "paths contain")} ${classPathEntry?html}: - <#if hideDependencyPathsByDefault> - <#-- The dependency paths are not summarized --> - - -

- - <#if hasRootCause> -

${dependencyPathRootCauses[classPathEntry]?html} -

- <#else> - -
    - <#list dependencyPaths as dependencyPath > -
  • ${dependencyPath}
    • - -
- - - -<#macro formatDependencyGraph graph node parent> - <#if node == graph.getRootPath() > - <#assign label = 'root' /> - <#else> - <#assign label = 'parent: ' + parent.getLeaf() /> - -

${node.getLeaf()}

-
    - <#list graph.getChildren(node) as childNode> - <#if node != childNode> -
  • - <@formatDependencyGraph graph childNode node /> -
    • - - -
- - -<#macro testResult row name> - <#if row.getResult(name)?? ><#-- checking isNotNull() --> - <#-- When it's not null, the test ran. It's either PASS or FAIL --> - <#assign test_label = row.getResult(name)?then('PASS', 'FAIL')> - <#assign failure_count = row.getFailureCount(name)> - <#else> - <#-- Null means there's an exception and test couldn't run --> - <#assign test_label = "UNAVAILABLE"> - - - <#if row.getResult(name)?? > - <#assign page_anchor = name?replace(" ", "-")?lower_case /> - - <#if failure_count gt 0>${pluralize(failure_count, "FAILURE", "FAILURES")} - <#else>PASS - - - <#else>UNAVAILABLE - - - - -<#macro pieChartSvg description ratio> - <#assign largeArcFlag = (ratio gt 0.5)?string("1", "0")> - <#assign endPointX = pieChart.calculateEndPointX(100, 100, 100, ratio)> - <#assign endPointY = pieChart.calculateEndPointY(100, 100, 100, ratio)> - - ${description} - - - - \ No newline at end of file diff --git a/dashboard/src/main/resources/templates/unstable_artifacts.ftl b/dashboard/src/main/resources/templates/unstable_artifacts.ftl deleted file mode 100644 index e02e56204b..0000000000 --- a/dashboard/src/main/resources/templates/unstable_artifacts.ftl +++ /dev/null @@ -1,40 +0,0 @@ - - - <#include "macros.ftl"> - - - Google Cloud Platform Java Open Source Dependency Dashboard: Unstable Artifacts - - - - -

Dependency Status of ${coordinates}

-

BOM: ${coordinates?html}

-

Pre 1.0 Versions

- -

- These are dependencies found in the GCP orbit that have not yet reached 1.0. - No 1.0 or later library should depend on them. - If the libraries are stable, advance them to 1.0. - Otherwise replace the dependency with something else. -

- - <#assign unstableCount = 0> -
    - <#list latestArtifacts as artifact, version> - <#if version[0] == '0'> - <#assign unstableCount++> -
  • ${artifact}:${version}
    • - - -
- - <#if unstableCount == 0> -

All versions are 1.0 or later.

- - - -
-

Last generated at ${lastUpdated}

- - \ No newline at end of file diff --git a/dashboard/src/main/resources/templates/version_index.ftl b/dashboard/src/main/resources/templates/version_index.ftl deleted file mode 100644 index 602bdf633a..0000000000 --- a/dashboard/src/main/resources/templates/version_index.ftl +++ /dev/null @@ -1,20 +0,0 @@ - - - - - ${groupId}:${artifactId} - - - -

${groupId}:${artifactId}

- -
    - <#list versions as version> -
  • - ${version} -
    • - -
- - - diff --git a/dashboard/src/test/java/com/google/cloud/tools/opensource/dashboard/ArtifactResultsTest.java b/dashboard/src/test/java/com/google/cloud/tools/opensource/dashboard/ArtifactResultsTest.java deleted file mode 100644 index bcfab9e49c..0000000000 --- a/dashboard/src/test/java/com/google/cloud/tools/opensource/dashboard/ArtifactResultsTest.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright 2018 Google LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.cloud.tools.opensource.dashboard; - -import org.eclipse.aether.artifact.DefaultArtifact; -import org.junit.Assert; -import org.junit.Test; - -public class ArtifactResultsTest { - - private ArtifactResults results = - new ArtifactResults(new DefaultArtifact("com.google.guava:guava:23.0")); - - @Test - public void testAddResult() { - results.addResult("foo", 0); - results.addResult("bar", 10); - Assert.assertTrue(results.getResult("foo")); - Assert.assertFalse(results.getResult("bar")); - Assert.assertNull(results.getResult("baz")); - } - - @Test - public void testGetFailureCount() { - results.addResult("foo", 0); - results.addResult("bar", 10); - Assert.assertEquals(0, results.getFailureCount("foo")); - Assert.assertEquals(10, results.getFailureCount("bar")); - Assert.assertEquals(0, results.getFailureCount("baz")); - } - - @Test - public void testGetCoordinates() { - Assert.assertEquals("com.google.guava:guava:23.0", results.getCoordinates()); - } - -} diff --git a/dashboard/src/test/java/com/google/cloud/tools/opensource/dashboard/BomTest.java b/dashboard/src/test/java/com/google/cloud/tools/opensource/dashboard/BomTest.java deleted file mode 100644 index 56bd491bca..0000000000 --- a/dashboard/src/test/java/com/google/cloud/tools/opensource/dashboard/BomTest.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright 2019 Google LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.cloud.tools.opensource.dashboard; - -import java.io.IOException; -import java.net.HttpURLConnection; -import java.net.URL; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.List; - -import org.eclipse.aether.artifact.Artifact; -import org.junit.Assert; -import org.junit.Test; -import com.google.cloud.tools.opensource.dependencies.Bom; -import com.google.cloud.tools.opensource.dependencies.MavenRepositoryException; - -public class BomTest { - - @Test - public void testLtsBom() - throws IOException, MavenRepositoryException { - Path bomPath = Paths.get("..", "boms", "cloud-lts-bom", "pom.xml").toAbsolutePath(); - checkBom(bomPath); - } - - @Test - public void testLibrariesBom() - throws IOException, MavenRepositoryException { - Path bomPath = Paths.get("..", "boms", "cloud-oss-bom", "pom.xml").toAbsolutePath(); - checkBom(bomPath); - } - - private void checkBom(Path bomPath) throws MavenRepositoryException, IOException { - List artifacts = Bom.readBom(bomPath).getManagedDependencies(); - for (Artifact artifact : artifacts) { - assertReachable(buildMavenCentralUrl(artifact)); - } - } - - private static String buildMavenCentralUrl(Artifact artifact) { - return "https://repo1.maven.org/maven2/" - + artifact.getGroupId().replace('.', '/') - + "/" - + artifact.getArtifactId() - + "/" - + artifact.getVersion() - + "/"; - } - - private static void assertReachable(String url) throws IOException { - HttpURLConnection connection = (HttpURLConnection) new URL(url).openConnection(); - connection.setRequestMethod("HEAD"); - try { - Assert.assertEquals( - "Could not reach " + url, HttpURLConnection.HTTP_OK, connection.getResponseCode()); - } catch (IOException ex) { - Assert.fail("Could not reach " + url + "\n" + ex.getMessage()); - } - } -} diff --git a/dashboard/src/test/java/com/google/cloud/tools/opensource/dashboard/DashboardArgumentsTest.java b/dashboard/src/test/java/com/google/cloud/tools/opensource/dashboard/DashboardArgumentsTest.java deleted file mode 100644 index 62204552b6..0000000000 --- a/dashboard/src/test/java/com/google/cloud/tools/opensource/dashboard/DashboardArgumentsTest.java +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Copyright 2019 Google LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.cloud.tools.opensource.dashboard; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; - -import com.google.cloud.tools.opensource.dashboard.DashboardArguments.DependencyMediationAlgorithm; -import com.google.common.truth.Truth; -import java.nio.file.Paths; -import org.apache.commons.cli.AlreadySelectedException; -import org.apache.commons.cli.MissingOptionException; -import org.apache.commons.cli.ParseException; -import org.junit.Assert; -import org.junit.Test; - -public class DashboardArgumentsTest { - - @Test - public void testParseArgument_file() throws ParseException { - DashboardArguments dashboardArguments = DashboardArguments.readCommandLine("-f", "../pom.xml"); - - assertTrue(dashboardArguments.hasFile()); - assertEquals(dashboardArguments.getDependencyMediation(), DependencyMediationAlgorithm.MAVEN); - assertEquals(Paths.get("../pom.xml").toAbsolutePath(), dashboardArguments.getBomFile()); - assertNull(dashboardArguments.getBomCoordinates()); - } - - @Test - public void testParseArgument_coordinates() throws ParseException { - DashboardArguments dashboardArguments = - DashboardArguments.readCommandLine("-c", "com.google.cloud:libraries-bom:1.0.0"); - assertFalse(dashboardArguments.hasFile()); - assertEquals( - "com.google.cloud:libraries-bom:1.0.0", dashboardArguments.getBomCoordinates()); - } - - @Test - public void testParseArgument_coordinatesWithLeadingSpace() throws ParseException { - // Maven exec plugin adds a leading space when arguments are passed as - // -Dexec.arguments="-c com.google.cloud:libraries-bom:$VERSION" - DashboardArguments dashboardArguments = - DashboardArguments.readCommandLine("-c", " com.google.cloud:libraries-bom:1.0.0"); - assertEquals( - "com.google.cloud:libraries-bom:1.0.0", dashboardArguments.getBomCoordinates()); - assertNull(dashboardArguments.getBomFile()); - } - - @Test - public void testParseArgument_missingInput() throws ParseException { - try { - DashboardArguments.readCommandLine(); - Assert.fail("The argument should validate missing input"); - } catch (MissingOptionException ex) { - // pass - } - } - - @Test - public void testParseArgument_duplicateOptions_coordinates_file() throws ParseException { - try { - DashboardArguments.readCommandLine( - "-c", "com.google.cloud:libraries-bom:1.0.0", "-f", "../pom.xml"); - Assert.fail("The argument should validate duplicate input"); - } catch (AlreadySelectedException ex) { - // pass - } - } - - @Test - public void testParseArgument_duplicateOptions_coordinates_allVersions() throws ParseException { - try { - DashboardArguments.readCommandLine( - "-c", "com.google.cloud:libraries-bom:1.0.0", "-a", "com.google.cloud:libraries-bom"); - Assert.fail("The argument should validate duplicate input"); - } catch (AlreadySelectedException ex) { - // pass - } - } - - @Test - public void testParseArgument_dependencyMediation_valid() throws ParseException { - DashboardArguments dashboardArguments = - DashboardArguments.readCommandLine("-f", "../pom.xml", "-m", "gradle"); - - assertEquals(dashboardArguments.getDependencyMediation(), DependencyMediationAlgorithm.GRADLE); - } - - @Test - public void testParseArgument_dependencyMediation_invalid() { - try { - DashboardArguments dashboardArguments = - DashboardArguments.readCommandLine("-f", "../pom.xml", "-m", "ant"); - Assert.fail("The argument should validate invalid dependency mediation value"); - } catch (ParseException ex) { - // pass - Truth.assertThat(ex.getMessage()).isEqualTo("Valid values for '-m' are [maven, gradle]"); - } - } -} diff --git a/dashboard/src/test/java/com/google/cloud/tools/opensource/dashboard/DashboardTest.java b/dashboard/src/test/java/com/google/cloud/tools/opensource/dashboard/DashboardTest.java deleted file mode 100644 index ae8e8dbdcb..0000000000 --- a/dashboard/src/test/java/com/google/cloud/tools/opensource/dashboard/DashboardTest.java +++ /dev/null @@ -1,484 +0,0 @@ -/* - * Copyright 2018 Google LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.cloud.tools.opensource.dashboard; - -import static com.google.common.collect.ImmutableList.toImmutableList; -import static com.google.common.truth.Truth.assertThat; -import static com.google.common.truth.Truth.assertWithMessage; - -import com.google.cloud.tools.opensource.dashboard.DashboardArguments.DependencyMediationAlgorithm; -import com.google.cloud.tools.opensource.dependencies.Artifacts; -import com.google.cloud.tools.opensource.dependencies.Bom; -import com.google.common.base.CharMatcher; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Streams; -import com.google.common.io.MoreFiles; -import com.google.common.io.RecursiveDeleteOption; -import com.google.common.truth.Correspondence; -import java.io.IOException; -import java.io.InputStream; -import java.net.URLDecoder; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.List; -import nu.xom.Builder; -import nu.xom.Document; -import nu.xom.Element; -import nu.xom.Node; -import nu.xom.Nodes; -import nu.xom.ParsingException; -import nu.xom.XPathContext; -import org.eclipse.aether.artifact.Artifact; -import org.eclipse.aether.artifact.DefaultArtifact; -import org.eclipse.aether.resolution.ArtifactDescriptorException; -import org.hamcrest.core.StringStartsWith; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.Assume; -import org.junit.BeforeClass; -import org.junit.Test; - -public class DashboardTest { - - private static final Correspondence NODE_VALUES = - Correspondence.transforming(node -> trimAndCollapseWhiteSpace(node.getValue()), "has value"); - - private static String trimAndCollapseWhiteSpace(String value) { - return CharMatcher.whitespace().trimAndCollapseFrom(value, ' '); - } - - private static Path outputDirectory; - private static Builder builder = new Builder(); - private static Document dashboard; - private static Document details; - private static Document unstable; - - @BeforeClass - public static void setUp() throws IOException, ParsingException { - // Creates "index.html" and artifact reports in outputDirectory - try { - outputDirectory = - DashboardMain.generate( - "com.google.cloud:libraries-bom:1.0.0", DependencyMediationAlgorithm.MAVEN); - } catch (Throwable t) { - t.printStackTrace(); - Assert.fail("Could not generate dashboard"); - } - - dashboard = parseOutputFile("index.html"); - details = parseOutputFile("artifact_details.html"); - unstable = parseOutputFile("unstable_artifacts.html"); - } - - @AfterClass - public static void cleanUp() { - try { - // Mac's APFS fails with InsecureRecursiveDeleteException without ALLOW_INSECURE. - // Still safe as this test does not use symbolic links - if (outputDirectory != null) { - MoreFiles.deleteRecursively(outputDirectory, RecursiveDeleteOption.ALLOW_INSECURE); - } - } catch (IOException ex) { - // no big deal - } - } - - @Test // https://github.com/GoogleCloudPlatform/cloud-opensource-java/issues/617 - public void testPlural() { - Nodes statisticItems = dashboard.query("//section[@class='statistics']/div/div"); - for (Node node : statisticItems) { - Node h2 = node.query("h2").get(0); - Node span = node.query("span").get(0); - int errorCount = Integer.parseInt(h2.getValue()); - if (errorCount == 1) { - String message = span.getValue(); - Assert.assertEquals("Has Linkage Errors", message); - } - } - - Assert.assertFalse(dashboard.toXML().contains("1 HAVE")); - } - - @Test - public void testHeader() { - Nodes h1 = dashboard.query("//h1"); - assertThat(h1).hasSize(1); - Assert.assertEquals("com.google.cloud:libraries-bom:1.0.0 Dependency Status", - h1.get(0).getValue()); - } - - @Test - public void testSvg() { - XPathContext context = new XPathContext("svg", "http://www.w3.org/2000/svg"); - Nodes svg = dashboard.query("//svg:svg", context); - assertThat(svg).hasSize(4); - } - - @Test - public void testCss() { - Path dashboardCss = outputDirectory.resolve("dashboard.css"); - Assert.assertTrue(Files.exists(dashboardCss)); - Assert.assertTrue(Files.isRegularFile(dashboardCss)); - } - - private static Document parseOutputFile(String fileName) - throws IOException, ParsingException { - Path html = outputDirectory.resolve(fileName); - Assert.assertTrue("Could not find a regular file for " + fileName, - Files.isRegularFile(html)); - Assert.assertTrue("The file is not readable: " + fileName, Files.isReadable(html)); - - try (InputStream source = Files.newInputStream(html)) { - return builder.build(source); - } - } - - @Test - public void testArtifactDetails() throws IOException, ArtifactDescriptorException { - List artifacts = Bom.readBom("com.google.cloud:libraries-bom:1.0.0") - .getManagedDependencies(); - Assert.assertTrue("Not enough artifacts found", artifacts.size() > 1); - - Assert.assertEquals("en-US", dashboard.getRootElement().getAttribute("lang").getValue()); - - Nodes tr = details.query("//tr"); - Assert.assertEquals(artifacts.size() + 1, tr.size()); // header row adds 1 - for (int i = 1; i < tr.size(); i++) { // start at 1 to skip header row - Nodes td = tr.get(i).query("td"); - Assert.assertEquals(Artifacts.toCoordinates(artifacts.get(i - 1)), td.get(0).getValue()); - for (int j = 1; j < 5; ++j) { // start at 1 to skip the leftmost artifact coordinates column - assertValidCellValue((Element) td.get(j)); - } - } - Nodes href = details.query("//tr/td[@class='artifact-name']/a/@href"); - for (int i = 0; i < href.size(); i++) { - String fileName = href.get(i).getValue(); - Artifact artifact = artifacts.get(i); - Assert.assertEquals( - Artifacts.toCoordinates(artifact).replace(':', '_') + ".html", - URLDecoder.decode(fileName, "UTF-8")); - Path componentReport = outputDirectory.resolve(fileName); - Assert.assertTrue(fileName + " is missing", Files.isRegularFile(componentReport)); - try { - Document report = builder.build(componentReport.toFile()); - Assert.assertEquals("en-US", report.getRootElement().getAttribute("lang").getValue()); - } catch (ParsingException ex) { - byte[] data = Files.readAllBytes(componentReport); - String message = "Could not parse " + componentReport + " at line " + - ex.getLineNumber() + ", column " + ex.getColumnNumber() + "\r\n"; - message += ex.getMessage() + "\r\n"; - message += new String(data, StandardCharsets.UTF_8); - Assert.fail(message); - } - } - } - - private static void assertValidCellValue(Element cellElement) { - String cellValue = cellElement.getValue().replaceAll("\\s", ""); - assertThat(cellValue).containsMatch("PASS|\\d+FAILURES?"); - assertWithMessage("It should not use plural for 1 item") - .that(cellValue) - .doesNotContainMatch("1 FAILURES"); - assertThat(cellElement.getAttributeValue("class")).isAnyOf("pass", "fail"); - } - - @Test - public void testDashboard_statisticBox() { - Nodes artifactCount = - dashboard.query("//div[@class='statistic-item statistic-item-green']/h2"); - Assert.assertTrue(artifactCount.size() > 0); - for (Node artifactCountElement : artifactCount) { - String value = artifactCountElement.getValue().trim(); - Assert.assertTrue(value, Integer.parseInt(value) > 0); - } - } - - @Test - public void testLinkageReports() { - Nodes reports = details.query("//p[@class='jar-linkage-report']"); - // appengine-api-sdk, shown as first item in linkage errors, has these errors - assertThat(trimAndCollapseWhiteSpace(reports.get(0).getValue())) - .isEqualTo("4 target classes causing linkage errors referenced from 4 source classes."); - - Nodes dependencyPaths = details.query("//p[@class='linkage-check-dependency-paths']"); - Node dependencyPathMessageOnProblem = dependencyPaths.get(dependencyPaths.size() - 1); - Assert.assertEquals( - "The following paths contain com.google.guava:guava-jdk5:13.0: ▶", - trimAndCollapseWhiteSpace(dependencyPathMessageOnProblem.getValue())); - - Node dependencyPathMessageOnSource = dependencyPaths.get(dependencyPaths.size() - 2); - Assert.assertEquals( - "The following paths contain com.google.guava:guava:27.1-android:", - trimAndCollapseWhiteSpace(dependencyPathMessageOnSource.getValue())); - } - - @Test - public void testDashboard_recommendedCoordinates() { - Nodes recommendedListItem = dashboard.query("//ul[@id='recommended']/li"); - Assert.assertTrue(recommendedListItem.size() > 100); - - // fails if these are not valid Maven coordinates - for (Node node : recommendedListItem) { - new DefaultArtifact(node.getValue()); - } - - ImmutableList coordinateList = - Streams.stream(recommendedListItem).map(Node::getValue).collect(toImmutableList()); - - ArrayList sorted = new ArrayList<>(coordinateList); - Comparator comparator = new SortWithoutVersion(); - Collections.sort(sorted, comparator); - - for (int i = 0; i < sorted.size(); i++) { - Assert.assertEquals( - "Coordinates are not sorted: ", sorted.get(i), coordinateList.get(i)); - } - } - - private static class SortWithoutVersion implements Comparator { - @Override - public int compare(String s1, String s2) { - s1 = s1.substring(0, s1.lastIndexOf(':')); - s2 = s2.substring(0, s2.lastIndexOf(':')); - return s1.compareTo(s2); - } - } - - @Test - public void testDashboard_unstableDependencies() { - // Pre 1.0 version section - Nodes li = unstable.query("//ul[@id='unstable']/li"); - Assert.assertTrue(li.size() > 1); - for (int i = 0; i < li.size(); i++) { - String value = li.get(i).getValue(); - Assert.assertTrue(value, value.contains(":0")); - } - - // This element appears only when every dependency becomes stable - Nodes stable = dashboard.query("//p[@id='stable-notice']"); - assertThat(stable).isEmpty(); - } - - @Test - public void testDashboard_lastUpdatedField() { - Nodes updated = dashboard.query("//p[@id='updated']"); - Assert.assertEquals( - "Could not find updated field: " + dashboard.toXML(), 1, updated.size()); - } - - @Test - public void testComponent_linkageCheckResult_java8() throws IOException, ParsingException { - Assume.assumeThat(System.getProperty("java.version"), StringStartsWith.startsWith("1.8.")); - // The version used in libraries-bom 1.0.0 - Document document = parseOutputFile( - "com.google.http-client_google-http-client-appengine_1.29.1.html"); - Nodes reports = document.query("//p[@class='jar-linkage-report']"); - assertThat(reports).hasSize(1); - assertThat(trimAndCollapseWhiteSpace(reports.get(0).getValue())) - .isEqualTo("100 target classes causing linkage errors referenced from 540 source classes."); - - Nodes causes = document.query("//p[@class='jar-linkage-report-cause']"); - assertWithMessage( - "google-http-client-appengine should show linkage errors for RpcStubDescriptor") - .that(causes) - .comparingElementsUsing(NODE_VALUES) - .contains( - "Class com.google.net.rpc3.client.RpcStubDescriptor is not found," - + " referenced from 21 classes ▶"); // '▶' is the toggle button - } - - @Test - public void testComponent_linkageCheckResult_java11() throws IOException, ParsingException { - String javaVersion = System.getProperty("java.version"); - // javaMajorVersion is 1 when we use Java 8. Still good indicator to ensure Java 11 or higher. - int javaMajorVersion = Integer.parseInt(javaVersion.split("\\.")[0]); - Assume.assumeTrue(javaMajorVersion >= 11); - - // The version used in libraries-bom 1.0.0. The google-http-client-appengine has been known to - // have linkage errors in its dependency appengine-api-1.0-sdk:1.9.71. - Document document = - parseOutputFile("com.google.http-client_google-http-client-appengine_1.29.1.html"); - Nodes reports = document.query("//p[@class='jar-linkage-report']"); - assertThat(reports).hasSize(1); - - // This number of linkage errors differs between Java 8 and Java 11 for the javax.activation - // package removal (JEP 320: Remove the Java EE and CORBA Modules). For the detail, see - // https://github.com/GoogleCloudPlatform/cloud-opensource-java/issues/1849. - assertThat(trimAndCollapseWhiteSpace(reports.get(0).getValue())) - .isEqualTo("105 target classes causing linkage errors referenced from 562 source classes."); - - Nodes causes = document.query("//p[@class='jar-linkage-report-cause']"); - assertWithMessage( - "google-http-client-appengine should show linkage errors for RpcStubDescriptor") - .that(causes) - .comparingElementsUsing(NODE_VALUES) - .contains( - "Class com.google.net.rpc3.client.RpcStubDescriptor is not found," - + " referenced from 21 classes ▶"); // '▶' is the toggle button - } - - @Test - public void testComponent_success() throws IOException, ParsingException { - Document document = parseOutputFile( - "com.google.api.grpc_proto-google-common-protos_1.14.0.html"); - Nodes greens = document.query("//h3[@style='color: green']"); - Assert.assertTrue(greens.size() >= 2); - Nodes presDependencyMediation = - document.query("//pre[@class='suggested-dependency-mediation']"); - // There's a pre tag for dependency - assertThat(presDependencyMediation).hasSize(1); - - Nodes presDependencyTree = document.query("//p[@class='dependency-tree-node']"); - Assert.assertTrue( - "Dependency Tree should be shown in dashboard", presDependencyTree.size() > 0); - } - - @Test - public void testComponent_failure() throws IOException, ParsingException { - Document document = parseOutputFile( - "com.google.api.grpc_grpc-google-common-protos_1.14.0.html"); - - // com.google.api.grpc:grpc-google-common-protos:1.14.0 has no green section - Nodes greens = document.query("//h3[@style='color: green']"); - assertThat(greens).isEmpty(); - - // "Global Upper Bounds Fixes", "Upper Bounds Fixes", and "Suggested Dependency Updates" are red - Nodes reds = document.query("//h3[@style='color: red']"); - assertThat(reds).hasSize(3); - Nodes presDependencyMediation = - document.query("//pre[@class='suggested-dependency-mediation']"); - Assert.assertTrue( - "For failed component, suggested dependency should be shown", - presDependencyMediation.size() >= 1); - Nodes dependencyTree = document.query("//p[@class='dependency-tree-node']"); - Assert.assertTrue( - "Dependency Tree should be shown in dashboard even when FAILED", - dependencyTree.size() > 0); - } - - @Test - public void testLinkageErrorsInProvidedDependency() throws IOException, ParsingException { - // google-http-client-appengine has provided dependency to (problematic) appengine-api-1.0-sdk - Document document = parseOutputFile( - "com.google.http-client_google-http-client-appengine_1.29.1.html"); - Nodes linkageCheckMessages = document.query("//ul[@class='jar-linkage-report-cause']/li"); - assertThat(linkageCheckMessages.size()).isGreaterThan(0); - assertThat(linkageCheckMessages.get(0).getValue()) - .contains("com.google.appengine.api.appidentity.AppIdentityServicePb"); - } - - @Test - public void testLinkageErrors_ensureNoDuplicateSymbols() throws IOException, ParsingException { - Document document = - parseOutputFile("com.google.http-client_google-http-client-appengine_1.29.1.html"); - Nodes linkageCheckMessages = document.query("//p[@class='jar-linkage-report-cause']"); - assertThat(linkageCheckMessages.size()).isGreaterThan(0); - - List messages = new ArrayList<>(); - for (int i = 0; i < linkageCheckMessages.size(); ++i) { - messages.add(linkageCheckMessages.get(i).getValue()); - } - - // When uniqueness of SymbolProblem and Symbol classes are incorrect, dashboard has duplicates. - assertThat(messages).containsNoDuplicates(); - } - - @Test - public void testZeroLinkageErrorShowsZero() throws IOException, ParsingException { - // grpc-auth does not have a linkage error, and it should show zero in the section - Document document = parseOutputFile("io.grpc_grpc-auth_1.20.0.html"); - Nodes linkageErrorsTotal = document.query("//p[@id='linkage-errors-total']"); - assertThat(linkageErrorsTotal).hasSize(1); - assertThat(linkageErrorsTotal.get(0).getValue()).contains("0 linkage error(s)"); - } - - @Test - public void testGlobalUpperBoundUpgradeMessage() throws IOException, ParsingException { - // Case 1: BOM needs to be updated - Document document = parseOutputFile("com.google.protobuf_protobuf-java-util_3.6.1.html"); - Nodes globalUpperBoundBomUpgradeNodes = - document.query("//li[@class='global-upper-bound-bom-upgrade']"); - assertThat(globalUpperBoundBomUpgradeNodes).hasSize(1); - String bomUpgradeMessage = globalUpperBoundBomUpgradeNodes.get(0).getValue(); - assertThat(bomUpgradeMessage) - .contains( - "Upgrade com.google.protobuf:protobuf-java-util:jar:3.6.1 in the BOM to version \"3.7.1\""); - - // Case 2: Dependency needs to be updated - Nodes globalUpperBoundDependencyUpgradeNodes = - document.query("//li[@class='global-upper-bound-dependency-upgrade']"); - - // The artifact report should contain the following 6 global upper bound dependency upgrades: - // Upgrade com.google.guava:guava:jar:19.0 to version "27.1-android" - // Upgrade com.google.protobuf:protobuf-java:jar:3.6.1 to version "3.7.1" - assertThat(globalUpperBoundDependencyUpgradeNodes.size()).isEqualTo(2); - String dependencyUpgradeMessage = globalUpperBoundDependencyUpgradeNodes.get(0).getValue(); - assertThat(dependencyUpgradeMessage) - .contains("Upgrade com.google.guava:guava:jar:19.0 to version \"27.1-android\""); - } - - @Test - public void testBomCoordinatesInComponent() throws IOException, ParsingException { - Document document = parseOutputFile("com.google.protobuf_protobuf-java-util_3.6.1.html"); - Nodes bomCoordinatesNodes = document.query("//p[@class='bom-coordinates']"); - assertThat(bomCoordinatesNodes).hasSize(1); - Assert.assertEquals( - "BOM: com.google.cloud:libraries-bom:1.0.0", bomCoordinatesNodes.get(0).getValue()); - } - - @Test - public void testBomCoordinatesInArtifactDetails() throws IOException, ParsingException { - Document document = parseOutputFile("artifact_details.html"); - Nodes bomCoordinatesNodes = document.query("//p[@class='bom-coordinates']"); - assertThat(bomCoordinatesNodes).hasSize(1); - Assert.assertEquals( - "BOM: com.google.cloud:libraries-bom:1.0.0", bomCoordinatesNodes.get(0).getValue()); - } - - @Test - public void testBomCoordinatesInUnstableArtifacts() throws IOException, ParsingException { - Document document = parseOutputFile("unstable_artifacts.html"); - Nodes bomCoordinatesNodes = document.query("//p[@class='bom-coordinates']"); - assertThat(bomCoordinatesNodes).hasSize(1); - Assert.assertEquals( - "BOM: com.google.cloud:libraries-bom:1.0.0", bomCoordinatesNodes.get(0).getValue()); - } - - @Test - public void testDependencyTrees() throws IOException, ParsingException { - Document document = parseOutputFile("dependency_trees.html"); - Nodes dependencyTreeParagraph = document.query("//p[@class='dependency-tree-node']"); - - // characterization test - assertThat(dependencyTreeParagraph).hasSize(38391); - Assert.assertEquals( - "com.google.protobuf:protobuf-java:jar:3.6.1", dependencyTreeParagraph.get(0).getValue()); - } - - @Test - public void testOutputDirectory() { - Assert.assertTrue( - "The dashboard should be created at target/com.google.cloud/libraries-bom/1.0.0", - outputDirectory.endsWith( - Paths.get("target", "com.google.cloud", "libraries-bom", "1.0.0"))); - } -} diff --git a/dashboard/src/test/java/com/google/cloud/tools/opensource/dashboard/DashboardUnavailableArtifactTest.java b/dashboard/src/test/java/com/google/cloud/tools/opensource/dashboard/DashboardUnavailableArtifactTest.java deleted file mode 100644 index 1d8f8998bf..0000000000 --- a/dashboard/src/test/java/com/google/cloud/tools/opensource/dashboard/DashboardUnavailableArtifactTest.java +++ /dev/null @@ -1,169 +0,0 @@ -/* - * Copyright 2018 Google LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.cloud.tools.opensource.dashboard; - -import com.google.cloud.tools.opensource.classpath.AnnotatedClassPath; -import com.google.cloud.tools.opensource.classpath.ClassPathResult; -import com.google.cloud.tools.opensource.dependencies.Artifacts; -import com.google.cloud.tools.opensource.dependencies.Bom; -import com.google.cloud.tools.opensource.dependencies.DependencyGraph; -import com.google.cloud.tools.opensource.dependencies.DependencyGraphBuilder; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.io.MoreFiles; -import com.google.common.io.RecursiveDeleteOption; -import com.google.common.truth.Truth; -import freemarker.template.Configuration; -import freemarker.template.TemplateException; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import nu.xom.Builder; -import nu.xom.Document; -import nu.xom.Element; -import nu.xom.Nodes; -import nu.xom.ParsingException; -import org.eclipse.aether.RepositoryException; -import org.eclipse.aether.artifact.Artifact; -import org.eclipse.aether.artifact.DefaultArtifact; -import org.eclipse.aether.graph.Dependency; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; - -public class DashboardUnavailableArtifactTest { - - private static Path outputDirectory; - private Bom bom = new Bom("test:test:1.2.4", null); - private Builder builder = new Builder(); - - @BeforeClass - public static void setUp() throws IOException { - outputDirectory = Files.createDirectories(Paths.get("target", "dashboard")); - } - - @AfterClass - public static void cleanUp() throws IOException { - // Mac's APFS fails with InsecureRecursiveDeleteException without ALLOW_INSECURE. - // Still safe as this test does not use symbolic links - MoreFiles.deleteRecursively(outputDirectory, RecursiveDeleteOption.ALLOW_INSECURE); - } - - @Test - public void testDashboardForRepositoryException() throws TemplateException { - Configuration configuration = DashboardMain.configureFreemarker(); - Artifact validArtifact = new DefaultArtifact("io.grpc:grpc-context:1.15.0"); - Artifact nonExistentArtifact = new DefaultArtifact("io.grpc:nonexistent:jar:1.15.0"); - DependencyGraphBuilder graphBuilder = new DependencyGraphBuilder(); - Map map = new LinkedHashMap<>(); - DependencyGraph graph1 = - graphBuilder.buildMavenDependencyGraph(new Dependency(validArtifact, "compile")); - DependencyGraph graph2 = - graphBuilder.buildMavenDependencyGraph(new Dependency(nonExistentArtifact, "compile")); - map.put(validArtifact, new ArtifactInfo(graph1, graph2)); - map.put(nonExistentArtifact, new ArtifactInfo(new RepositoryException("foo"))); - - ArtifactCache cache = new ArtifactCache(); - cache.setInfoMap(map); - List artifactResults = - DashboardMain.generateReports( - configuration, - outputDirectory, - cache, - ImmutableMap.of(), - new ClassPathResult(new AnnotatedClassPath(), ImmutableList.of()), - bom); - - Assert.assertEquals( - "The length of the ArtifactResults should match the length of artifacts", - 2, - artifactResults.size()); - Assert.assertEquals( - "The first artifact result should be valid", - true, - artifactResults.get(0).getResult(DashboardMain.TEST_NAME_UPPER_BOUND)); - ArtifactResults errorArtifactResult = artifactResults.get(1); - Assert.assertNull( - "The second artifact result should be null", - errorArtifactResult.getResult(DashboardMain.TEST_NAME_UPPER_BOUND)); - Assert.assertEquals( - "The error artifact result should contain error message", - "foo", - errorArtifactResult.getExceptionMessage()); - } - - @Test - public void testDashboardWithRepositoryException() - throws IOException, TemplateException, ParsingException { - Configuration configuration = DashboardMain.configureFreemarker(); - - Artifact validArtifact = new DefaultArtifact("io.grpc:grpc-context:1.15.0"); - ArtifactResults validArtifactResult = new ArtifactResults(validArtifact); - validArtifactResult.addResult(DashboardMain.TEST_NAME_UPPER_BOUND, 0); - validArtifactResult.addResult(DashboardMain.TEST_NAME_DEPENDENCY_CONVERGENCE, 0); - validArtifactResult.addResult(DashboardMain.TEST_NAME_GLOBAL_UPPER_BOUND, 0); - - Artifact invalidArtifact = new DefaultArtifact("io.grpc:nonexistent:jar:1.15.0"); - ArtifactResults errorArtifactResult = new ArtifactResults(invalidArtifact); - errorArtifactResult.setExceptionMessage( - "Could not find artifact io.grpc:nonexistent:jar:1.15.0 in central" - + " (https://repo1.maven.org/maven2/)"); - - List table = new ArrayList<>(); - table.add(validArtifactResult); - table.add(errorArtifactResult); - - DashboardMain.generateDashboard( - configuration, - outputDirectory, - table, - ImmutableList.of(), - ImmutableMap.of(), - new ClassPathResult(new AnnotatedClassPath(), ImmutableList.of()), - bom); - - Path generatedHtml = outputDirectory.resolve("artifact_details.html"); - Assert.assertTrue(Files.isRegularFile(generatedHtml)); - Document document = builder.build(generatedHtml.toFile()); - Assert.assertEquals("en-US", document.getRootElement().getAttribute("lang").getValue()); - Nodes tr = document.query("//tr"); - - Assert.assertEquals( - "The size of rows in table should match the number of artifacts + 1 (header)", - tr.size(),table.size() + 1); - - Nodes tdForValidArtifact = tr.get(1).query("td"); - Assert.assertEquals( - Artifacts.toCoordinates(validArtifact), tdForValidArtifact.get(0).getValue()); - Element firstResult = (Element) (tdForValidArtifact.get(2)); - Truth.assertThat(firstResult.getValue().trim()).isEqualTo("PASS"); - Truth.assertThat(firstResult.getAttributeValue("class")).isEqualTo("pass"); - - Nodes tdForErrorArtifact = tr.get(2).query("td"); - Assert.assertEquals( - Artifacts.toCoordinates(invalidArtifact), tdForErrorArtifact.get(0).getValue()); - Element secondResult = (Element) (tdForErrorArtifact.get(2)); - Truth.assertThat(secondResult.getValue().trim()).isEqualTo("UNAVAILABLE"); - Truth.assertThat(secondResult.getAttributeValue("class")).isEqualTo("unavailable"); - } -} diff --git a/dashboard/src/test/java/com/google/cloud/tools/opensource/dashboard/FreemarkerTest.java b/dashboard/src/test/java/com/google/cloud/tools/opensource/dashboard/FreemarkerTest.java deleted file mode 100644 index a499fa0552..0000000000 --- a/dashboard/src/test/java/com/google/cloud/tools/opensource/dashboard/FreemarkerTest.java +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Copyright 2019 Google LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.cloud.tools.opensource.dashboard; - -import com.google.cloud.tools.opensource.classpath.AnnotatedClassPath; -import com.google.cloud.tools.opensource.classpath.ClassFile; -import com.google.cloud.tools.opensource.classpath.ClassNotFoundProblem; -import com.google.cloud.tools.opensource.classpath.ClassPathEntry; -import com.google.cloud.tools.opensource.classpath.ClassPathResult; -import com.google.cloud.tools.opensource.classpath.ClassSymbol; -import com.google.cloud.tools.opensource.classpath.LinkageProblem; -import com.google.cloud.tools.opensource.dependencies.Bom; -import com.google.cloud.tools.opensource.dependencies.DependencyGraph; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; -import com.google.common.io.MoreFiles; -import com.google.common.io.RecursiveDeleteOption; -import com.google.common.truth.Truth; -import freemarker.template.Configuration; -import freemarker.template.TemplateException; -import java.io.File; -import java.io.IOException; -import java.net.URISyntaxException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.List; -import nu.xom.Builder; -import nu.xom.Document; -import nu.xom.Node; -import nu.xom.Nodes; -import nu.xom.ParsingException; -import org.eclipse.aether.artifact.Artifact; -import org.eclipse.aether.artifact.DefaultArtifact; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; - -/** - * Unit tests for FreeMarker logic without reading any JAR files. - */ -public class FreemarkerTest { - - private static Path outputDirectory; - private static ImmutableMap> symbolProblemTable; - - private Builder builder = new Builder(); - - @BeforeClass - public static void setUpDirectory() throws IOException { - outputDirectory = Files.createDirectories(Paths.get("target", "dashboard")); - } - - @Before - public void setUp() { - Artifact artifact = new DefaultArtifact("com.google:foo:1.0.0") - .setFile(new File("foo/bar-1.2.3.jar")); - ClassPathEntry entry = new ClassPathEntry(artifact); - ImmutableSet dummyProblems = - ImmutableSet.of( - new ClassNotFoundProblem( - new ClassFile(entry, "abc.def.G"), new ClassSymbol("com.foo.Bar"))); - symbolProblemTable = ImmutableMap.of(entry, dummyProblems); - } - - @AfterClass - public static void cleanUp() throws IOException { - // Mac's APFS fails with InsecureRecursiveDeleteException without ALLOW_INSECURE. - // Still safe as this test does not use symbolic links - MoreFiles.deleteRecursively(outputDirectory, RecursiveDeleteOption.ALLOW_INSECURE); - } - - @Test - public void testCountFailures() throws IOException, TemplateException, ParsingException { - Configuration configuration = DashboardMain.configureFreemarker(); - - Artifact artifact1 = new DefaultArtifact("io.grpc:grpc-context:1.15.0"); - ArtifactResults results1 = new ArtifactResults(artifact1); - results1.addResult("Linkage Errors", 56); - - Artifact artifact2 = new DefaultArtifact("grpc:grpc:1.15.0"); - ArtifactResults results2 = new ArtifactResults(artifact2); - results2.addResult("Linkage Errors", 0); - - List table = ImmutableList.of(results1, results2); - List globalDependencies = ImmutableList.of(); - DashboardMain.generateDashboard( - configuration, - outputDirectory, - table, - globalDependencies, - symbolProblemTable, - new ClassPathResult(new AnnotatedClassPath(), ImmutableList.of()), - new Bom("mock:artifact:1.6.7", null)); - - Path dashboardHtml = outputDirectory.resolve("index.html"); - Assert.assertTrue(Files.isRegularFile(dashboardHtml)); - Document document = builder.build(dashboardHtml.toFile()); - - // xom's query cannot specify partial class field, e.g., 'statistic-item' - Nodes counts = document.query("//div[@class='container']/div/h2"); - Assert.assertTrue(counts.size() > 0); - for (int i = 0; i < counts.size(); i++) { - Integer.parseInt(counts.get(i).getValue().trim()); - } - // Linkage Errors - Truth.assertThat(counts.get(1).getValue().trim()).isEqualTo("1"); - } - - @Test - public void testVersionIndex() - throws IOException, TemplateException, URISyntaxException, ParsingException { - Path output = - DashboardMain.generateVersionIndex( - "com.google.cloud", - "libraries-bom", - ImmutableList.of("1.0.0", "2.0.0", "2.1.0-SNAPSHOT")); - Assert.assertTrue( - output.endsWith(Paths.get("target", "com.google.cloud", "libraries-bom", "index.html"))); - Assert.assertTrue(Files.isRegularFile(output)); - - Document document = builder.build(output.toFile()); - Nodes links = document.query("//a/@href"); - Assert.assertEquals(3, links.size()); - Node snapshotLink = links.get(2); - // 2.1.0-SNAPSHOT has directory 'snapshot' - Assert.assertEquals("snapshot/index.html", snapshotLink.getValue()); - } -} diff --git a/dashboard/src/test/java/com/google/cloud/tools/opensource/dashboard/PieChartTest.java b/dashboard/src/test/java/com/google/cloud/tools/opensource/dashboard/PieChartTest.java deleted file mode 100644 index 177f4243c9..0000000000 --- a/dashboard/src/test/java/com/google/cloud/tools/opensource/dashboard/PieChartTest.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright 2019 Google LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.cloud.tools.opensource.dashboard; - -import java.awt.geom.Point2D; -import org.junit.Assert; -import org.junit.Test; - -public class PieChartTest { - - private static final double TOLERANCE = 0.0001; - - @Test - public void testRightAngle() { - Point2D actual = PieChart.calculateEndPoint(100, 100, 100, .25); - Point2D expected = new Point2D.Double(200, 100); - Assert.assertEquals(expected.getX(), actual.getX(), TOLERANCE); - Assert.assertEquals(expected.getY(), actual.getY(), TOLERANCE); - } - - @Test - public void testZero() { - Point2D actual = PieChart.calculateEndPoint(100, 100, 100, 0); - Point2D expected = new Point2D.Double(100, 0); - Assert.assertEquals(expected.getX(), actual.getX(), TOLERANCE); - Assert.assertEquals(expected.getY(), actual.getY(), TOLERANCE); - } - - @Test - public void testOneEighth() { - int radius = 100; - double ratio = 1.0/8.0; - Point2D actual = PieChart.calculateEndPoint(radius, 100, 100, ratio); - Point2D expected = new Point2D.Double(100 + 100 / Math.sqrt(2), 100 - 100 / Math.sqrt(2)); - Assert.assertEquals(expected.getX(), actual.getX(), TOLERANCE); - Assert.assertEquals(expected.getY(), actual.getY(), TOLERANCE); - } - - @Test - public void testHalf() { - Point2D actual = PieChart.calculateEndPoint(100, 100, 100, 0.5); - Point2D expected = new Point2D.Double(100, 200); - Assert.assertEquals(expected.getX(), actual.getX(), TOLERANCE); - Assert.assertEquals(expected.getY(), actual.getY(), TOLERANCE); - } - - @Test - public void testOffCenter() { - Point2D actual = PieChart.calculateEndPoint(150, 150, 100, 0.5); - Point2D expected = new Point2D.Double(150, 250); - Assert.assertEquals(expected.getX(), actual.getX(), TOLERANCE); - Assert.assertEquals(expected.getY(), actual.getY(), TOLERANCE); - } - - @Test - public void testFull() { - Point2D actual = PieChart.calculateEndPoint(100, 100, 100, 1.0); - Point2D expected = new Point2D.Double(100, 0); - Assert.assertEquals(expected.getX(), actual.getX(), TOLERANCE); - Assert.assertEquals(expected.getY(), actual.getY(), TOLERANCE); - } - - @Test - public void testMoreThanFull() { - Point2D actual = PieChart.calculateEndPoint(100, 100, 100, 5.5); - Point2D expected = new Point2D.Double(100, 0); - Assert.assertEquals(expected.getX(), actual.getX(), TOLERANCE); - Assert.assertEquals(expected.getY(), actual.getY(), TOLERANCE); - } - -} - - - diff --git a/dependencies/README.md b/dependencies/README.md index c265ab122d..585e9e5b84 100644 --- a/dependencies/README.md +++ b/dependencies/README.md @@ -5,7 +5,7 @@ Linkage Checker is a tool that finds [linkage errors]( path. It scans the class files in the class path for references to other classes and reports any reference that cannot be satisfied. -#### User Documentation +### User Documentation Linkage Checker can be used from Maven or Gradle. diff --git a/dependencies/pom.xml b/dependencies/pom.xml index 80185114c8..69bd85b9af 100644 --- a/dependencies/pom.xml +++ b/dependencies/pom.xml @@ -8,7 +8,7 @@ com.google.cloud.tools dependencies-parent - 1.5.12-SNAPSHOT + 1.5.16-SNAPSHOT dependencies @@ -46,11 +46,24 @@ org.apache.bcel bcel - 6.5.0 + 6.6.1 org.apache.maven maven-core + + + org.apache.maven.shared + maven-shared-utils + + + + + org.apache.maven.shared + maven-shared-utils + + 3.3.3 @@ -122,18 +135,16 @@ - java8-incompatible-reference-check + exec-linkage-checker org.codehaus.mojo exec-maven-plugin + 3.5.0 false - com.google.cloud.tools.opensource.dependencies.Java8IncompatibleReferenceCheck - - ../boms/cloud-oss-bom/pom.xml - + com.google.cloud.tools.opensource.classpath.LinkageCheckerMain diff --git a/dependencies/src/main/java/com/google/cloud/tools/opensource/classpath/ClassDumper.java b/dependencies/src/main/java/com/google/cloud/tools/opensource/classpath/ClassDumper.java index 45564253f6..82115b7e03 100644 --- a/dependencies/src/main/java/com/google/cloud/tools/opensource/classpath/ClassDumper.java +++ b/dependencies/src/main/java/com/google/cloud/tools/opensource/classpath/ClassDumper.java @@ -89,7 +89,7 @@ static ClassDumper create(List entries) throws IOException { .collect(toImmutableList()); checkArgument( unreadableFiles.isEmpty(), "Some jar files are not readable: %s", unreadableFiles); - + Map map = new HashMap<>(); for (ClassPathEntry entry : entries) { for (String className : entry.getFileNames()) { @@ -98,7 +98,7 @@ static ClassDumper create(List entries) throws IOException { } } } - + return new ClassDumper(entries, extensionClassLoader, map); } @@ -141,37 +141,54 @@ static boolean isArrayClass(String className) { return className.startsWith("["); } - /** - * Returns a map from classes to the symbol references they contain. - */ + /** Returns a map from classes to the symbol references they contain. */ SymbolReferences findSymbolReferences() throws IOException { SymbolReferences.Builder builder = new SymbolReferences.Builder(); for (ClassPathEntry jar : inputClassPath) { + int totalClassFileCount = 0; + int incompatibleClassFileCount = 0; for (JavaClass javaClass : listClasses(jar)) { + totalClassFileCount++; if (isCompatibleClassFileVersion(javaClass)) { String className = javaClass.getClassName(); // In listClasses(jar), ClassPathRepository creates JavaClass through the first JAR file // that contains the class. It may be different from "jar" for an overlapping class. ClassFile source = new ClassFile(findClassLocation(className), className); builder.addAll(findSymbolReferences(source, javaClass)); + } else { + incompatibleClassFileCount++; } } + if (incompatibleClassFileCount > 0) { + logger.warning( + String.format( + "%s has %d (out of %d) incompatible class files (class file major version is outside %d <= v <= %d).", + jar, + incompatibleClassFileCount, + totalClassFileCount, + MINIMUM_CLASS_FILE_MAJOR_VERSION, + MAXIMUM_CLASS_FILE_MAJOR_VERSION)); + } } return builder.build(); } + private static final int MINIMUM_CLASS_FILE_MAJOR_VERSION = 45; + private static final int MAXIMUM_CLASS_FILE_MAJOR_VERSION = 52; + /** - * Returns true if {@code javaClass} file format is compatible with this tool. Currently - * Java 8 and earlier are supported. + * Returns true if {@code javaClass} file format is compatible with this tool. Currently Java 8 + * (class file major version 52) and earlier are supported. * * @see Java * Virtual Machine Specification: The ClassFile Structure: minor_version, major_version */ private static boolean isCompatibleClassFileVersion(JavaClass javaClass) { int classFileMajorVersion = javaClass.getMajor(); - return 45 <= classFileMajorVersion && classFileMajorVersion <= 52; + return MINIMUM_CLASS_FILE_MAJOR_VERSION <= classFileMajorVersion + && classFileMajorVersion <= MAXIMUM_CLASS_FILE_MAJOR_VERSION; } private static SymbolReferences.Builder findSymbolReferences( @@ -182,7 +199,7 @@ private static SymbolReferences.Builder findSymbolReferences( Constant[] constants = constantPool.getConstantPool(); for (Constant constant : constants) { if (constant == null) { - continue; + continue; } byte constantTag = constant.getTag(); @@ -262,8 +279,7 @@ private static ClassSymbol makeSymbol( return new ClassSymbol(targetClassName); } - private static MethodSymbol makeSymbol( - ConstantCP constantMethodref, ConstantPool constantPool) { + private static MethodSymbol makeSymbol(ConstantCP constantMethodref, ConstantPool constantPool) { String className = constantMethodref.getClass(constantPool); ConstantNameAndType constantNameAndType = constantNameAndType(constantMethodref, constantPool); String methodName = constantNameAndType.getName(constantPool); @@ -303,7 +319,7 @@ static ImmutableSet listInnerClassNames(JavaClass javaClass) { continue; } } - + // Class names stored in constant pool have '/' as separator. We want '.' (as binary name) String normalInnerClassName = innerClassName.replace('/', '.'); innerClassNames.add(normalInnerClassName); diff --git a/dependencies/src/main/java/com/google/cloud/tools/opensource/classpath/LinkageCheckResultException.java b/dependencies/src/main/java/com/google/cloud/tools/opensource/classpath/LinkageCheckResultException.java index d7acd68a89..3e66a5ab7d 100644 --- a/dependencies/src/main/java/com/google/cloud/tools/opensource/classpath/LinkageCheckResultException.java +++ b/dependencies/src/main/java/com/google/cloud/tools/opensource/classpath/LinkageCheckResultException.java @@ -22,7 +22,7 @@ *

The caller of the tool can tell the existence of linkage errors by checking the exit status of * the {@link LinkageCheckerMain}. */ -final class LinkageCheckResultException extends Exception { +public final class LinkageCheckResultException extends Exception { LinkageCheckResultException(int linkageErrorCount) { super( "Found " diff --git a/dependencies/src/main/java/com/google/cloud/tools/opensource/classpath/LinkageChecker.java b/dependencies/src/main/java/com/google/cloud/tools/opensource/classpath/LinkageChecker.java index 12dfc715de..4e4f74b572 100644 --- a/dependencies/src/main/java/com/google/cloud/tools/opensource/classpath/LinkageChecker.java +++ b/dependencies/src/main/java/com/google/cloud/tools/opensource/classpath/LinkageChecker.java @@ -36,6 +36,7 @@ import java.util.Queue; import java.util.Set; import java.util.logging.Logger; +import java.util.stream.Collectors; import javax.annotation.Nullable; import org.apache.bcel.classfile.Field; import org.apache.bcel.classfile.FieldOrMethod; @@ -54,6 +55,7 @@ public class LinkageChecker { private final ImmutableList classPath; private final SymbolReferences symbolReferences; private final ClassReferenceGraph classReferenceGraph; + private final List sourceFilterList; private final ExcludedErrors excludedErrors; @VisibleForTesting @@ -66,7 +68,7 @@ public ClassReferenceGraph getClassReferenceGraph() { } public static LinkageChecker create(List classPath) throws IOException { - return create(classPath, ImmutableSet.copyOf(classPath), null); + return create(classPath, ImmutableSet.copyOf(classPath), ImmutableList.of(), null); } /** @@ -79,6 +81,7 @@ public static LinkageChecker create(List classPath) throws IOExc public static LinkageChecker create( List classPath, Iterable entryPoints, + List sourceFilterList, @Nullable Path exclusionFile) throws IOException { Preconditions.checkArgument(!classPath.isEmpty(), "The linkage classpath is empty."); @@ -93,6 +96,7 @@ public static LinkageChecker create( classPath, symbolReferenceMaps, classReferenceGraph, + sourceFilterList, ExcludedErrors.create(exclusionFile)); } @@ -129,13 +133,13 @@ public static LinkageChecker create(Bom bom, Path exclusionFile) List artifactsInBom = classpath.subList(0, managedDependencies.size()); ImmutableSet entryPoints = ImmutableSet.copyOf(artifactsInBom); - return LinkageChecker.create(classpath, entryPoints, exclusionFile); + return LinkageChecker.create(classpath, entryPoints, ImmutableList.of(), exclusionFile); } @VisibleForTesting LinkageChecker cloneWith(SymbolReferences newSymbolMaps) { return new LinkageChecker( - classDumper, classPath, newSymbolMaps, classReferenceGraph, excludedErrors); + classDumper, classPath, newSymbolMaps, classReferenceGraph, ImmutableList.of(), excludedErrors); } private LinkageChecker( @@ -143,14 +147,28 @@ private LinkageChecker( List classPath, SymbolReferences symbolReferenceMaps, ClassReferenceGraph classReferenceGraph, + List sourceFilterList, ExcludedErrors excludedErrors) { this.classDumper = Preconditions.checkNotNull(classDumper); this.classPath = ImmutableList.copyOf(classPath); this.classReferenceGraph = Preconditions.checkNotNull(classReferenceGraph); this.symbolReferences = Preconditions.checkNotNull(symbolReferenceMaps); + this.sourceFilterList = sourceFilterList; this.excludedErrors = Preconditions.checkNotNull(excludedErrors); } + /** + * Two artifacts are considered equal if only the Maven Coordinates (GAV) are equal. This is + * included instead of using Artifact.equals() because the `equals()` implementation + * of DefaultArtifact checks more fields than just the GAV Coordinates (also checks the classifier, + * file path, properties, etc are all equal). + */ + private boolean areArtifactsEquals(Artifact artifact1, Artifact artifact2) { + return artifact1.getGroupId().equals(artifact2.getGroupId()) + && artifact1.getArtifactId().equals(artifact2.getArtifactId()) + && artifact1.getVersion().equals(artifact2.getVersion()); + } + /** * Searches the classpath for linkage errors. * @@ -161,7 +179,21 @@ public ImmutableSet findLinkageProblems() throws IOException { ImmutableSet.Builder problemToClass = ImmutableSet.builder(); // This sourceClassFile is a source of references to other symbols. - for (ClassFile classFile : symbolReferences.getClassFiles()) { + Set classFiles = symbolReferences.getClassFiles(); + + // Filtering the classFiles from the JARs (instead of using the problem filter) has additional a few + // additional benefits. 1. Reduces the total amount of linkage references to match and 2. Doesn't require + // an exclusion file to know all the possible flaky or false positive problems + if (!sourceFilterList.isEmpty()) { + // Filter the list to only contain class files that come from the classes we are interested in. + // Run through each class file and check that the class file's corresponding artifact matches + // any artifact specified in the sourceFilterList + classFiles = classFiles.stream() + .filter(x -> sourceFilterList.stream() + .anyMatch(y -> areArtifactsEquals(x.getClassPathEntry().getArtifact(), y))) + .collect(Collectors.toSet()); + } + for (ClassFile classFile : classFiles) { ImmutableSet classSymbols = symbolReferences.getClassSymbols(classFile); for (ClassSymbol classSymbol : classSymbols) { if (classSymbol instanceof SuperClassSymbol) { @@ -202,7 +234,7 @@ public ImmutableSet findLinkageProblems() throws IOException { } } - for (ClassFile classFile : symbolReferences.getClassFiles()) { + for (ClassFile classFile : classFiles) { ImmutableSet methodSymbols = symbolReferences.getMethodSymbols(classFile); ImmutableSet classFileNames = classFile.getClassPathEntry().getFileNames(); for (MethodSymbol methodSymbol : methodSymbols) { @@ -215,7 +247,7 @@ public ImmutableSet findLinkageProblems() throws IOException { } } - for (ClassFile classFile : symbolReferences.getClassFiles()) { + for (ClassFile classFile : classFiles) { ImmutableSet fieldSymbols = symbolReferences.getFieldSymbols(classFile); ImmutableSet classFileNames = classFile.getClassPathEntry().getFileNames(); for (FieldSymbol fieldSymbol : fieldSymbols) { diff --git a/dependencies/src/main/java/com/google/cloud/tools/opensource/classpath/LinkageCheckerArguments.java b/dependencies/src/main/java/com/google/cloud/tools/opensource/classpath/LinkageCheckerArguments.java index 04a5556f86..1696fe095d 100644 --- a/dependencies/src/main/java/com/google/cloud/tools/opensource/classpath/LinkageCheckerArguments.java +++ b/dependencies/src/main/java/com/google/cloud/tools/opensource/classpath/LinkageCheckerArguments.java @@ -25,6 +25,9 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; import org.apache.commons.cli.DefaultParser; @@ -124,6 +127,17 @@ private static Options configureOptions() { .build(); inputGroup.addOption(jarOption); + Option sourceFilter = + Option.builder("s") + .longOpt("source-filter") + .hasArgs() + .valueSeparator(',') + .desc("List of maven coordinates for artifacts (separated by ','). " + + "These are artifacts whose class files are searched as the source of the " + + "potential Linkage Errors.") + .build(); + options.addOption(sourceFilter); + Option repositoryOption = Option.builder("m") .longOpt("maven-repositories") @@ -277,4 +291,18 @@ Path getOutputExclusionFile() { } return null; } + + /** + * Returns a list of artifacts to search where Linkage Errors stem from. If the argument is not + * specified, return an empty List. + */ + List getSourceFilterArtifactList() { + if (commandLine.hasOption("s")) { + String[] mavenCoordinatesOption = commandLine.getOptionValues("s"); + return Arrays.stream(mavenCoordinatesOption) + .map(DefaultArtifact::new) + .collect(Collectors.toList()); + } + return ImmutableList.of(); + } } diff --git a/dependencies/src/main/java/com/google/cloud/tools/opensource/classpath/LinkageCheckerMain.java b/dependencies/src/main/java/com/google/cloud/tools/opensource/classpath/LinkageCheckerMain.java index 73102c3174..834b872cd2 100644 --- a/dependencies/src/main/java/com/google/cloud/tools/opensource/classpath/LinkageCheckerMain.java +++ b/dependencies/src/main/java/com/google/cloud/tools/opensource/classpath/LinkageCheckerMain.java @@ -33,7 +33,7 @@ /** * A tool to find linkage errors in a class path. */ -class LinkageCheckerMain { +public class LinkageCheckerMain { /** * Forms a classpath from Maven coordinates or a list of jar files and reports linkage errors in @@ -133,7 +133,7 @@ private static Problems checkJarFiles( ImmutableSet entryPoints = ImmutableSet.copyOf(inputClassPath); LinkageChecker linkageChecker = LinkageChecker.create( - inputClassPath, entryPoints, linkageCheckerArguments.getInputExclusionFile()); + inputClassPath, entryPoints, ImmutableList.of(), linkageCheckerArguments.getInputExclusionFile()); ImmutableSet linkageProblems = findLinkageProblems(linkageChecker, linkageCheckerArguments.getReportOnlyReachable()); @@ -161,7 +161,7 @@ private static Problems checkArtifacts( LinkageChecker linkageChecker = LinkageChecker.create( - inputClassPath, entryPoints, linkageCheckerArguments.getInputExclusionFile()); + inputClassPath, entryPoints, linkageCheckerArguments.getSourceFilterArtifactList(), linkageCheckerArguments.getInputExclusionFile()); ImmutableSet linkageProblems = findLinkageProblems(linkageChecker, linkageCheckerArguments.getReportOnlyReachable()); diff --git a/dependencies/src/main/java/com/google/cloud/tools/opensource/classpath/LinkageProblem.java b/dependencies/src/main/java/com/google/cloud/tools/opensource/classpath/LinkageProblem.java index ceb25c3034..1eadf1133a 100644 --- a/dependencies/src/main/java/com/google/cloud/tools/opensource/classpath/LinkageProblem.java +++ b/dependencies/src/main/java/com/google/cloud/tools/opensource/classpath/LinkageProblem.java @@ -218,10 +218,12 @@ public static String formatLinkageProblems( for (AbstractMethodProblem abstractMethodProblem : abstractMethodProblems.build()) { output.append(abstractMethodProblem + "\n"); - output.append(" Cause:\n"); LinkageProblemCause cause = abstractMethodProblem.getCause(); - String causeWithIndent = cause.toString().replaceAll("\n", "\n "); - output.append(" " + causeWithIndent + "\n"); + if (cause != null) { + output.append(" Cause:\n"); + String causeWithIndent = cause.toString().replaceAll("\n", "\n "); + output.append(" " + causeWithIndent + "\n"); + } } if (classPathResult != null) { diff --git a/dependencies/src/main/java/com/google/cloud/tools/opensource/classpath/LinkageProblemCauseAnnotator.java b/dependencies/src/main/java/com/google/cloud/tools/opensource/classpath/LinkageProblemCauseAnnotator.java index 4b1242f04e..c54be34144 100644 --- a/dependencies/src/main/java/com/google/cloud/tools/opensource/classpath/LinkageProblemCauseAnnotator.java +++ b/dependencies/src/main/java/com/google/cloud/tools/opensource/classpath/LinkageProblemCauseAnnotator.java @@ -19,13 +19,17 @@ import static com.google.common.base.Preconditions.checkNotNull; import com.google.cloud.tools.opensource.dependencies.DependencyPath; +import com.google.common.collect.ImmutableList; import java.io.IOException; import java.util.HashMap; import java.util.Map; +import java.util.logging.Logger; import org.eclipse.aether.artifact.Artifact; /** Annotates {@link LinkageProblem}s with {@link LinkageProblemCause}s. */ public final class LinkageProblemCauseAnnotator { + private static final Logger logger = + Logger.getLogger(LinkageProblemCauseAnnotator.class.getName()); private LinkageProblemCauseAnnotator() {} @@ -35,74 +39,100 @@ private LinkageProblemCauseAnnotator() {} * @param classPathBuilder class path builder to resolve dependency graphs * @param rootResult the class path used for generating the linkage problems * @param linkageProblems linkage problems to annotate - * @throws IOException when there is a problem reading JAR files */ public static void annotate( ClassPathBuilder classPathBuilder, ClassPathResult rootResult, - Iterable linkageProblems) - throws IOException { + Iterable linkageProblems) { checkNotNull(classPathBuilder); checkNotNull(rootResult); checkNotNull(linkageProblems); - Map cache = new HashMap<>(); + Map artifactResolutionCache = new HashMap<>(); for (LinkageProblem linkageProblem : linkageProblems) { - ClassFile sourceClass = linkageProblem.getSourceClass(); - ClassPathEntry sourceEntry = sourceClass.getClassPathEntry(); + // Annotating linkage errors is a nice-to-have feature for Linkage Checker plugins. Let's not + // fail the entire process if there are problems, such as classPathBuilder unable to resolve + // one artifact or to return correct dependency tree. + try { + annotateProblem(classPathBuilder, rootResult, artifactResolutionCache, linkageProblem); + } catch (Exception ex) { + logger.warning("Failed to annotate: " + linkageProblem); + linkageProblem.setCause(UnknownCause.getInstance()); + } + } + } - Artifact sourceArtifact = sourceEntry.getArtifact(); + private static void annotateProblem( + ClassPathBuilder classPathBuilder, + ClassPathResult rootResult, + Map artifactResolutionCache, + LinkageProblem linkageProblem) + throws IOException { + ClassFile sourceClass = linkageProblem.getSourceClass(); + ClassPathEntry sourceEntry = sourceClass.getClassPathEntry(); - ClassPathResult subtreeResult = cache.get(sourceArtifact); - if (subtreeResult == null) { - // Resolves the dependency graph with the source artifact at the root. - subtreeResult = classPathBuilder.resolveWithMaven(sourceArtifact); - cache.put(sourceArtifact, subtreeResult); - } + Artifact sourceArtifact = sourceEntry.getArtifact(); - Symbol symbol = linkageProblem.getSymbol(); - ClassPathEntry entryInSubtree = subtreeResult.findEntryBySymbol(symbol); - if (entryInSubtree == null) { - linkageProblem.setCause(UnknownCause.getInstance()); - } else { - Artifact artifactInSubtree = entryInSubtree.getArtifact(); - DependencyPath pathToSourceEntry = rootResult.getDependencyPaths(sourceEntry).get(0); - DependencyPath pathFromSourceEntryToUnselectedEntry = - subtreeResult.getDependencyPaths(entryInSubtree).get(0); - DependencyPath pathToUnselectedEntry = - pathToSourceEntry.concat(pathFromSourceEntryToUnselectedEntry); + ClassPathResult subtreeResult = artifactResolutionCache.get(sourceArtifact); + if (subtreeResult == null) { + // Resolves the dependency graph with the source artifact at the root. + subtreeResult = classPathBuilder.resolveWithMaven(sourceArtifact); + artifactResolutionCache.put(sourceArtifact, subtreeResult); + } - ClassPathEntry selectedEntry = - rootResult.findEntryById( - artifactInSubtree.getGroupId(), artifactInSubtree.getArtifactId()); - if (selectedEntry != null) { - Artifact selectedArtifact = selectedEntry.getArtifact(); - if (!selectedArtifact.getVersion().equals(artifactInSubtree.getVersion())) { - // Different version of that artifact is selected in rootResult - linkageProblem.setCause( - new DependencyConflict( - linkageProblem, - rootResult.getDependencyPaths(selectedEntry).get(0), - pathToUnselectedEntry)); - } else { - // A linkage error was already there when sourceArtifact was built. - linkageProblem.setCause(UnknownCause.getInstance()); - } - } else { - // No artifact that matches groupId and artifactId in rootResult. + Symbol symbol = linkageProblem.getSymbol(); + ClassPathEntry entryInSubtree = subtreeResult.findEntryBySymbol(symbol); + if (entryInSubtree == null) { + linkageProblem.setCause(UnknownCause.getInstance()); + return; + } + Artifact artifactInSubtree = entryInSubtree.getArtifact(); + ImmutableList dependencyPathsToSource = + rootResult.getDependencyPaths(sourceEntry); + if (dependencyPathsToSource.isEmpty()) { + // When an artifact is excluded, it's possible to have no dependency path to sourceEntry. + linkageProblem.setCause(UnknownCause.getInstance()); + return; + } + DependencyPath pathToSourceEntry = dependencyPathsToSource.get(0); + DependencyPath pathFromSourceEntryToUnselectedEntry = + subtreeResult.getDependencyPaths(entryInSubtree).get(0); + DependencyPath pathToUnselectedEntry = + pathToSourceEntry.concat(pathFromSourceEntryToUnselectedEntry); - // Checking exclusion elements in the dependency path - Artifact excludingArtifact = - pathToSourceEntry.findExclusion( - artifactInSubtree.getGroupId(), artifactInSubtree.getArtifactId()); - if (excludingArtifact != null) { - linkageProblem.setCause( - new ExcludedDependency(pathToUnselectedEntry, excludingArtifact)); - } else { - linkageProblem.setCause(new MissingDependency(pathToUnselectedEntry)); - } + ClassPathEntry selectedEntry = + rootResult.findEntryById( + artifactInSubtree.getGroupId(), artifactInSubtree.getArtifactId()); + if (selectedEntry != null) { + Artifact selectedArtifact = selectedEntry.getArtifact(); + if (!selectedArtifact.getVersion().equals(artifactInSubtree.getVersion())) { + // Different version of that artifact is selected in rootResult + ImmutableList pathToSelectedArtifact = + rootResult.getDependencyPaths(selectedEntry); + if (pathToSelectedArtifact.isEmpty()) { + linkageProblem.setCause(UnknownCause.getInstance()); + return; } + linkageProblem.setCause( + new DependencyConflict( + linkageProblem, pathToSelectedArtifact.get(0), pathToUnselectedEntry)); + } else { + // A linkage error was already there when sourceArtifact was built. + linkageProblem.setCause(UnknownCause.getInstance()); + } + } else { + // No artifact that matches groupId and artifactId in rootResult. + + // Checking exclusion elements in the dependency path + Artifact excludingArtifact = + pathToSourceEntry.findExclusion( + artifactInSubtree.getGroupId(), artifactInSubtree.getArtifactId()); + if (excludingArtifact != null) { + linkageProblem.setCause(new ExcludedDependency(pathToUnselectedEntry, excludingArtifact)); + } else { + linkageProblem.setCause(new MissingDependency(pathToUnselectedEntry)); } } } + } diff --git a/dependencies/src/main/java/com/google/cloud/tools/opensource/dependencies/Bom.java b/dependencies/src/main/java/com/google/cloud/tools/opensource/dependencies/Bom.java index c0eda3e0d6..3d35310444 100644 --- a/dependencies/src/main/java/com/google/cloud/tools/opensource/dependencies/Bom.java +++ b/dependencies/src/main/java/com/google/cloud/tools/opensource/dependencies/Bom.java @@ -19,7 +19,6 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; import java.nio.file.Path; import java.util.ArrayList; import java.util.List; @@ -38,9 +37,6 @@ import org.eclipse.aether.resolution.ArtifactDescriptorResult; public final class Bom { - - private static final ImmutableSet BOM_SKIP_ARTIFACT_IDS = - ImmutableSet.of("google-cloud-logging-logback", "google-cloud-contrib"); private final ImmutableList artifacts; private final String coordinates; @@ -151,12 +147,7 @@ public static boolean shouldSkipBomMember(Artifact artifact) { if ("test-jar".equals(type)) { return true; } - - // TODO remove this hack once we get these out of google-cloud-java's BOM - if (BOM_SKIP_ARTIFACT_IDS.contains(artifact.getArtifactId())) { - return true; - } - + return false; } } diff --git a/dependencies/src/main/resources/linkage-checker-exclusion-default.xml b/dependencies/src/main/resources/linkage-checker-exclusion-default.xml index 742570d89f..2b6de4bc9e 100644 --- a/dependencies/src/main/resources/linkage-checker-exclusion-default.xml +++ b/dependencies/src/main/resources/linkage-checker-exclusion-default.xml @@ -10,11 +10,18 @@ https://github.com/GoogleCloudPlatform/cloud-opensource-java/issues/816 - - - - + + + + + GraalVM-related libraries depend on Java Compiler Interface (JVMCI) that + only exists in special JDK. These missing classes are false positives, because + the code is only invoked when running in a GraalVM. + https://github.com/GoogleCloudPlatform/cloud-opensource-java/issues/929 + + + @@ -29,17 +36,16 @@ - - - - - - + + Substrate VM is part of GraalVM and the virtual machine references classes in the JVM-internal + package 'sun.text.normalizer' that are not available in JVM runtime. + https://github.com/GoogleCloudPlatform/cloud-opensource-java/issues/1825 + @@ -404,4 +410,49 @@ https://github.com/GoogleCloudPlatform/cloud-opensource-java/issues/1871 + + + + + + + + + Flogger's JavaLangAccessStackGetter and StackWalkerStackGetter trys to use available classes + in different JDK versions. Certain classes are unavailable in a JDK. + + + + + + + + + + + Flogger's JavaLangAccessStackGetter and StackWalkerStackGetter trys to use available classes + in different JDK versions. Certain classes are unavailable in a JDK. + + + + + + + + Google Cloud libraries do not use SMTP protocols. There are discrepancy in + javax.mail:mail:1.4.3 com.sun.mail:javax.mail:1.6.2 in dependency graph. + + + + + + + + + + + Shaded class in com.fasterxml.woodstox:woodstox-core:6.2.6 has a class + reference to the textui.Driver but the shading process didn't include it. + + diff --git a/dependencies/src/test/java/com/google/cloud/tools/opensource/classpath/ClassPathBuilderTest.java b/dependencies/src/test/java/com/google/cloud/tools/opensource/classpath/ClassPathBuilderTest.java index 86d645e640..3104f444f9 100644 --- a/dependencies/src/test/java/com/google/cloud/tools/opensource/classpath/ClassPathBuilderTest.java +++ b/dependencies/src/test/java/com/google/cloud/tools/opensource/classpath/ClassPathBuilderTest.java @@ -108,7 +108,8 @@ public void testBomToPaths_firstElementsAreBomMembers() throws RepositoryExcepti .isEqualTo("com.google.api:api-common:1.7.0"); // first element in the BOM int bomSize = managedDependencies.size(); String lastFileName = entries.get(bomSize - 1).toString(); - assertThat(lastFileName).isEqualTo("com.google.api:gax-httpjson:0.57.0"); // last element in BOM + assertThat(lastFileName) + .isEqualTo("com.google.code.findbugs:jsr305:3.0.2"); // last element in BOM } @Test diff --git a/dependencies/src/test/java/com/google/cloud/tools/opensource/classpath/ExclusionFilesIntegrationTest.java b/dependencies/src/test/java/com/google/cloud/tools/opensource/classpath/ExclusionFilesIntegrationTest.java index a0431398c5..c8bdf99fb4 100644 --- a/dependencies/src/test/java/com/google/cloud/tools/opensource/classpath/ExclusionFilesIntegrationTest.java +++ b/dependencies/src/test/java/com/google/cloud/tools/opensource/classpath/ExclusionFilesIntegrationTest.java @@ -58,7 +58,7 @@ public void testExclusion() classPathBuilder.resolve(ImmutableList.of(artifact), false, DependencyMediation.MAVEN); LinkageChecker linkagechecker = - LinkageChecker.create(classPathResult.getClassPath(), ImmutableList.of(), exclusionFile); + LinkageChecker.create(classPathResult.getClassPath(), ImmutableList.of(), ImmutableList.of(), exclusionFile); ImmutableSet linkageProblems = linkagechecker.findLinkageProblems(); Truth.assertThat(linkageProblems).isEmpty(); diff --git a/dependencies/src/test/java/com/google/cloud/tools/opensource/classpath/ExclusionFilesTest.java b/dependencies/src/test/java/com/google/cloud/tools/opensource/classpath/ExclusionFilesTest.java index 197466eb45..cd8245bc7d 100644 --- a/dependencies/src/test/java/com/google/cloud/tools/opensource/classpath/ExclusionFilesTest.java +++ b/dependencies/src/test/java/com/google/cloud/tools/opensource/classpath/ExclusionFilesTest.java @@ -340,14 +340,15 @@ public void testWriteExclusionFile_indent() ExclusionFiles.write(output, linkageErrors); - String actual = new String(Files.readAllBytes(output)); + String actual = new String(Files.readAllBytes(output)).replaceAll("\\R", "\n"); String expected = new String( - Files.readAllBytes( - absolutePathOfResource( - "exclusion-sample-rules/expected-exclusion-output-file.xml")), - StandardCharsets.UTF_8); + Files.readAllBytes( + absolutePathOfResource( + "exclusion-sample-rules/expected-exclusion-output-file.xml")), + StandardCharsets.UTF_8) + .replaceAll("\\R", "\n"); assertEquals(expected, actual); } diff --git a/dependencies/src/test/java/com/google/cloud/tools/opensource/classpath/LinkageCheckerMainIntegrationTest.java b/dependencies/src/test/java/com/google/cloud/tools/opensource/classpath/LinkageCheckerMainIntegrationTest.java index 231e63928e..992f544656 100644 --- a/dependencies/src/test/java/com/google/cloud/tools/opensource/classpath/LinkageCheckerMainIntegrationTest.java +++ b/dependencies/src/test/java/com/google/cloud/tools/opensource/classpath/LinkageCheckerMainIntegrationTest.java @@ -138,7 +138,7 @@ public void testBom_java8() LinkageCheckerMain.main(new String[] {"-b", "com.google.cloud:libraries-bom:1.0.0"}); fail("LinkageCheckerMain should throw LinkageCheckResultException upon errors"); } catch (LinkageCheckResultException expected) { - assertEquals("Found 631 linkage errors", expected.getMessage()); + assertEquals("Found 734 linkage errors", expected.getMessage()); } String output = readCapturedStdout(); @@ -172,7 +172,7 @@ public void testBom_java11() LinkageCheckerMain.main(new String[] {"-b", "com.google.cloud:libraries-bom:1.0.0"}); fail("LinkageCheckerMain should throw LinkageCheckResultException upon errors"); } catch (LinkageCheckResultException expected) { - assertEquals("Found 653 linkage errors", expected.getMessage()); + assertEquals("Found 758 linkage errors", expected.getMessage()); } String output = readCapturedStdout(); diff --git a/dependencies/src/test/java/com/google/cloud/tools/opensource/classpath/LinkageCheckerTest.java b/dependencies/src/test/java/com/google/cloud/tools/opensource/classpath/LinkageCheckerTest.java index df2912d2d5..96556c330f 100644 --- a/dependencies/src/test/java/com/google/cloud/tools/opensource/classpath/LinkageCheckerTest.java +++ b/dependencies/src/test/java/com/google/cloud/tools/opensource/classpath/LinkageCheckerTest.java @@ -1340,4 +1340,32 @@ public void testProtectedMethodsOfObject() throws Exception { "has linkage errors that reference symbols on class")) .doesNotContain("java.lang.Object"); } + + @Test + public void testSourceFilter() throws InvalidVersionSpecificationException, IOException { + // BQ-Storage v3.9.3 contains 3 known binary incompatibilities with Protobuf-Java 4.x + DefaultArtifact sourceArtifact = new DefaultArtifact("com.google.cloud:google-cloud-bigquerystorage:3.9.3"); + ClassPathResult classPathResult = + new ClassPathBuilder() + .resolve( + ImmutableList.of( + sourceArtifact, + new DefaultArtifact("com.google.protobuf:protobuf-java:4.27.4"), + new DefaultArtifact("com.google.protobuf:protobuf-java-util:4.27.4")), + false, + DependencyMediation.MAVEN); + + ImmutableList classPath = classPathResult.getClassPath(); + LinkageChecker linkageChecker = LinkageChecker.create( + classPath, + ImmutableSet.copyOf(classPath), + ImmutableList.of(sourceArtifact), + null); + + // Without a source-filter to narrow down the linkage errors that stem from BQ-Storage, Linkage Checker + // would report 119 errors. Narrowing it down with the source filter will only report the 3 known binary + // incompatibilities + ImmutableSet problems = linkageChecker.findLinkageProblems(); + Truth.assertThat(problems.size()).isEqualTo(3); + } } diff --git a/dependencies/src/test/java/com/google/cloud/tools/opensource/classpath/LinkageProblemCauseAnnotatorTest.java b/dependencies/src/test/java/com/google/cloud/tools/opensource/classpath/LinkageProblemCauseAnnotatorTest.java index 710548e67f..e63480629f 100644 --- a/dependencies/src/test/java/com/google/cloud/tools/opensource/classpath/LinkageProblemCauseAnnotatorTest.java +++ b/dependencies/src/test/java/com/google/cloud/tools/opensource/classpath/LinkageProblemCauseAnnotatorTest.java @@ -198,4 +198,26 @@ public void testAnnotate_dependencyInSpringRepository() throws IOException, Repo "org.reactivestreams:reactive-streams:1.0.3", Artifacts.toCoordinates(conflict.getPathToArtifactThruSource().getLeaf())); } + + @Test + public void testAnnotate_missingArtifactInTree() throws IOException, RepositoryException { + // The dependency tree of the google-api-client does not contain class "com.Foo" + ClassPathBuilder builder = new ClassPathBuilder(); + ClassPathResult classPathResult = + builder.resolve( + ImmutableList.of(new DefaultArtifact("com.google.api-client:google-api-client:1.27.0")), + false, + DependencyMediation.MAVEN); + + ClassPathEntry googleApiClient = classPathResult.getClassPath().get(0); + ClassNotFoundProblem dummyProblem = + new ClassNotFoundProblem( + new ClassFile(googleApiClient, "com.Foo"), new ClassSymbol("com.Bar")); + + LinkageProblemCauseAnnotator.annotate( + classPathBuilder, classPathResult, ImmutableSet.of(dummyProblem)); + + LinkageProblemCause cause = dummyProblem.getCause(); + assertTrue(cause instanceof UnknownCause); + } } diff --git a/dependencies/src/test/java/com/google/cloud/tools/opensource/dependencies/BomTest.java b/dependencies/src/test/java/com/google/cloud/tools/opensource/dependencies/BomTest.java index 29f32f7a41..200d7e8ca3 100644 --- a/dependencies/src/test/java/com/google/cloud/tools/opensource/dependencies/BomTest.java +++ b/dependencies/src/test/java/com/google/cloud/tools/opensource/dependencies/BomTest.java @@ -39,7 +39,7 @@ public void testReadBom_coordinates() throws ArtifactDescriptorException { List managedDependencies = bom.getManagedDependencies(); // Characterization test. As long as the artifact doesn't change (and it shouldn't) // the answer won't change. - Assert.assertEquals(134, managedDependencies.size()); + Assert.assertEquals(136, managedDependencies.size()); Assert.assertEquals("com.google.cloud:google-cloud-bom:0.61.0-alpha", bom.getCoordinates()); } diff --git a/dependencies/versions.txt b/dependencies/versions.txt new file mode 100644 index 0000000000..18c2d98c61 --- /dev/null +++ b/dependencies/versions.txt @@ -0,0 +1,4 @@ +# Format: +# module:released-version:current-version + +dependencies:1.5.15:1.5.16-SNAPSHOT diff --git a/docs/JLBP-0015.md b/docs/JLBP-0015.md index 6c6dddec1f..7dbf8aaa36 100644 --- a/docs/JLBP-0015.md +++ b/docs/JLBP-0015.md @@ -47,7 +47,7 @@ dependencies in its `` section to ensure that its build is consistent, but these dependency versions shouldn't be imported by consumers who import the BOM. -Example BOM: [google-cloud-bom](https://github.com/GoogleCloudPlatform/google-cloud-java/blob/master/google-cloud-bom/pom.xml). +Example BOM: [google-cloud-bom](https://github.com/googleapis/java-cloud-bom/blob/main/pom.xml). ## Libraries built with Gradle diff --git a/enforcer-rules/README.md b/enforcer-rules/README.md index 80e364243b..42c89389fb 100644 --- a/enforcer-rules/README.md +++ b/enforcer-rules/README.md @@ -17,8 +17,13 @@ $ mvn verify Listening for transport dt_socket at address: 5005 ``` -When you debug one of the integration tests in the "src/it" directory, use the following -command to specify the test case and to provide the debug parameter to Maven invoker plugin. +When you debug one of the integration tests in the "src/it" directory, check the +`build.log` files in the `enforcer-rules/target/it` directory (run +`find enforcer-rules -name 'build.log'`). +The file is used in verification scripts and usually contains build errors. + +If you want to attach a debugger, use the following command to specify the test +case and to provide the debug parameter to Maven invoker plugin. ``` mvn install -Dmaven.test.skip -Dinvoker.test=bom-project-using-spring-repository \ diff --git a/enforcer-rules/pom.xml b/enforcer-rules/pom.xml index e4239e9f4b..6f4147ac24 100644 --- a/enforcer-rules/pom.xml +++ b/enforcer-rules/pom.xml @@ -6,7 +6,7 @@ com.google.cloud.tools dependencies-parent - 1.5.12-SNAPSHOT + 1.5.16-SNAPSHOT linkage-checker-enforcer-rules @@ -48,7 +48,7 @@ - 3.0.0-M3 + 3.5.0 1.8 1.8 @@ -132,23 +132,51 @@ - maven-invoker-plugin - 3.2.2 - - ${project.build.directory}/it - ${project.build.directory}/local-repo - verify - + + org.eclipse.sisu + sisu-maven-plugin + 0.9.0.M1 - integration-test - install - run + main-index + + + invoker-integration-test + + + !performRelease + + + + + + maven-invoker-plugin + 3.2.2 + + ${project.build.directory}/it + ${project.build.directory}/local-repo + verify + + + + integration-test + + install + run + + + + + + + + diff --git a/enforcer-rules/src/it/abstract-method-errors/pom.xml b/enforcer-rules/src/it/abstract-method-errors/pom.xml index 13163944c4..c0f89086d7 100644 --- a/enforcer-rules/src/it/abstract-method-errors/pom.xml +++ b/enforcer-rules/src/it/abstract-method-errors/pom.xml @@ -52,7 +52,7 @@ org.apache.maven.plugins maven-enforcer-plugin - 3.0.0-M3 + @enforcer.version@ com.google.cloud.tools @@ -69,9 +69,7 @@ - - + diff --git a/enforcer-rules/src/it/bom-project-error/pom.xml b/enforcer-rules/src/it/bom-project-error/pom.xml index a2067b092a..673ab39bd8 100644 --- a/enforcer-rules/src/it/bom-project-error/pom.xml +++ b/enforcer-rules/src/it/bom-project-error/pom.xml @@ -54,7 +54,7 @@ org.apache.maven.plugins maven-enforcer-plugin - 3.0.0-M3 + @enforcer.version@ com.google.cloud.tools @@ -71,10 +71,9 @@ - + DEPENDENCY_MANAGEMENT - + diff --git a/enforcer-rules/src/it/bom-project-no-error/pom.xml b/enforcer-rules/src/it/bom-project-no-error/pom.xml index 8d2eb3767f..11e41ed0b3 100644 --- a/enforcer-rules/src/it/bom-project-no-error/pom.xml +++ b/enforcer-rules/src/it/bom-project-no-error/pom.xml @@ -50,7 +50,7 @@ org.apache.maven.plugins maven-enforcer-plugin - 3.0.0-M3 + @enforcer.version@ com.google.cloud.tools diff --git a/enforcer-rules/src/it/bom-project-no-packaging/pom.xml b/enforcer-rules/src/it/bom-project-no-packaging/pom.xml index 387a516efc..40549d43eb 100644 --- a/enforcer-rules/src/it/bom-project-no-packaging/pom.xml +++ b/enforcer-rules/src/it/bom-project-no-packaging/pom.xml @@ -48,7 +48,7 @@ org.apache.maven.plugins maven-enforcer-plugin - 3.0.0-M3 + @enforcer.version@ com.google.cloud.tools @@ -65,10 +65,9 @@ - + DEPENDENCY_MANAGEMENT - + diff --git a/enforcer-rules/src/it/bom-project-using-spring-repository/pom.xml b/enforcer-rules/src/it/bom-project-using-spring-repository/pom.xml index a734c6e5d4..47429492f5 100644 --- a/enforcer-rules/src/it/bom-project-using-spring-repository/pom.xml +++ b/enforcer-rules/src/it/bom-project-using-spring-repository/pom.xml @@ -67,7 +67,7 @@ org.apache.maven.plugins maven-enforcer-plugin - 3.0.0-M3 + @enforcer.version@ com.google.cloud.tools @@ -84,10 +84,9 @@ - + DEPENDENCY_MANAGEMENT - + diff --git a/enforcer-rules/src/it/bom-project-using-spring-repository/verify.groovy b/enforcer-rules/src/it/bom-project-using-spring-repository/verify.groovy index 1c46d86b5c..ce90a31ff6 100644 --- a/enforcer-rules/src/it/bom-project-using-spring-repository/verify.groovy +++ b/enforcer-rules/src/it/bom-project-using-spring-repository/verify.groovy @@ -14,6 +14,6 @@ assert !buildLog.text.contains("NullPointerException") // 4 linkage errors are references to java.util.concurrent.Flow class, which does not exist in // Java 8 runtime yet. -def expectedErrorCount = System.getProperty("java.version").startsWith("1.8.") ? 798 : 794 +def expectedErrorCount = System.getProperty("java.version").startsWith("1.8.") ? 111 : 108 assert buildLog.text.contains("Linkage Checker rule found $expectedErrorCount errors:") diff --git a/enforcer-rules/src/it/fail-build-for-linkage-errors/pom.xml b/enforcer-rules/src/it/fail-build-for-linkage-errors/pom.xml index 744213021f..1f3936fd00 100644 --- a/enforcer-rules/src/it/fail-build-for-linkage-errors/pom.xml +++ b/enforcer-rules/src/it/fail-build-for-linkage-errors/pom.xml @@ -50,7 +50,7 @@ org.apache.maven.plugins maven-enforcer-plugin - 3.0.0-M3 + @enforcer.version@ com.google.cloud.tools @@ -67,9 +67,7 @@ - - + diff --git a/enforcer-rules/src/it/inaccessible-class-error/pom.xml b/enforcer-rules/src/it/inaccessible-class-error/pom.xml index e002ae31c8..2ac107a3e3 100644 --- a/enforcer-rules/src/it/inaccessible-class-error/pom.xml +++ b/enforcer-rules/src/it/inaccessible-class-error/pom.xml @@ -52,7 +52,7 @@ org.apache.maven.plugins maven-enforcer-plugin - 3.0.0-M3 + @enforcer.version@ com.google.cloud.tools @@ -69,9 +69,7 @@ - - + diff --git a/enforcer-rules/src/it/missing-filter-file-error/pom.xml b/enforcer-rules/src/it/missing-filter-file-error/pom.xml index d0ba489324..452b37cf43 100644 --- a/enforcer-rules/src/it/missing-filter-file-error/pom.xml +++ b/enforcer-rules/src/it/missing-filter-file-error/pom.xml @@ -50,7 +50,7 @@ org.apache.maven.plugins maven-enforcer-plugin - 3.0.0-M3 + @enforcer.version@ com.google.cloud.tools diff --git a/enforcer-rules/src/it/report-only-reachable-bom/pom.xml b/enforcer-rules/src/it/report-only-reachable-bom/pom.xml index 2fbec94a2f..d26886752f 100644 --- a/enforcer-rules/src/it/report-only-reachable-bom/pom.xml +++ b/enforcer-rules/src/it/report-only-reachable-bom/pom.xml @@ -59,7 +59,7 @@ org.apache.maven.plugins maven-enforcer-plugin - 3.0.0-M3 + @enforcer.version@ com.google.cloud.tools @@ -76,11 +76,10 @@ - + DEPENDENCY_MANAGEMENT true - + diff --git a/enforcer-rules/src/it/report-only-reachable/pom.xml b/enforcer-rules/src/it/report-only-reachable/pom.xml index 4a73f04618..f7949a142c 100644 --- a/enforcer-rules/src/it/report-only-reachable/pom.xml +++ b/enforcer-rules/src/it/report-only-reachable/pom.xml @@ -51,7 +51,7 @@ org.apache.maven.plugins maven-enforcer-plugin - 3.0.0-M3 + @enforcer.version@ com.google.cloud.tools @@ -68,10 +68,9 @@ - + true - + diff --git a/enforcer-rules/src/it/return-type-mismatch/pom.xml b/enforcer-rules/src/it/return-type-mismatch/pom.xml index 0d0948b72c..f002511ce7 100644 --- a/enforcer-rules/src/it/return-type-mismatch/pom.xml +++ b/enforcer-rules/src/it/return-type-mismatch/pom.xml @@ -47,7 +47,7 @@ org.apache.maven.plugins maven-enforcer-plugin - 3.0.0-M3 + @enforcer.version@ com.google.cloud.tools @@ -64,9 +64,7 @@ - - + diff --git a/enforcer-rules/src/it/test-scope/pom.xml b/enforcer-rules/src/it/test-scope/pom.xml index 9a06e22261..7713b68dfb 100644 --- a/enforcer-rules/src/it/test-scope/pom.xml +++ b/enforcer-rules/src/it/test-scope/pom.xml @@ -56,7 +56,7 @@ org.apache.maven.plugins maven-enforcer-plugin - 3.0.0-M3 + @enforcer.version@ com.google.cloud.tools @@ -73,10 +73,9 @@ - + true - + diff --git a/enforcer-rules/src/it/war-project-private-modifier/pom.xml b/enforcer-rules/src/it/war-project-private-modifier/pom.xml index d30a4b2012..b0917c13ea 100644 --- a/enforcer-rules/src/it/war-project-private-modifier/pom.xml +++ b/enforcer-rules/src/it/war-project-private-modifier/pom.xml @@ -46,7 +46,7 @@ org.apache.maven.plugins maven-enforcer-plugin - 3.0.0-M3 + @enforcer.version@ com.google.cloud.tools diff --git a/enforcer-rules/src/main/java/com/google/cloud/tools/dependencies/enforcer/LinkageCheckerRule.java b/enforcer-rules/src/main/java/com/google/cloud/tools/dependencies/enforcer/LinkageCheckerRule.java index cd7cba74cc..cd69e51907 100644 --- a/enforcer-rules/src/main/java/com/google/cloud/tools/dependencies/enforcer/LinkageCheckerRule.java +++ b/enforcer-rules/src/main/java/com/google/cloud/tools/dependencies/enforcer/LinkageCheckerRule.java @@ -48,24 +48,23 @@ import java.util.List; import java.util.Map; import java.util.Set; -import javax.annotation.Nonnull; +import javax.inject.Inject; +import javax.inject.Named; import org.apache.maven.RepositoryUtils; +import org.apache.maven.enforcer.rule.api.AbstractEnforcerRule; +import org.apache.maven.enforcer.rule.api.EnforcerLogger; import org.apache.maven.enforcer.rule.api.EnforcerRuleException; -import org.apache.maven.enforcer.rule.api.EnforcerRuleHelper; import org.apache.maven.execution.MavenSession; import org.apache.maven.plugin.MojoExecution; -import org.apache.maven.plugin.logging.Log; -import org.apache.maven.plugins.enforcer.AbstractNonCacheableEnforcerRule; import org.apache.maven.project.DefaultDependencyResolutionRequest; import org.apache.maven.project.DependencyResolutionException; import org.apache.maven.project.DependencyResolutionRequest; import org.apache.maven.project.DependencyResolutionResult; import org.apache.maven.project.MavenProject; import org.apache.maven.project.ProjectDependenciesResolver; -import org.codehaus.plexus.component.configurator.expression.ExpressionEvaluationException; -import org.codehaus.plexus.component.repository.exception.ComponentLookupException; import org.eclipse.aether.DefaultRepositoryCache; import org.eclipse.aether.DefaultRepositorySystemSession; +import org.eclipse.aether.RepositorySystem; import org.eclipse.aether.RepositorySystemSession; import org.eclipse.aether.artifact.Artifact; import org.eclipse.aether.artifact.ArtifactTypeRegistry; @@ -79,7 +78,8 @@ import org.eclipse.aether.version.InvalidVersionSpecificationException; /** Linkage Checker Maven Enforcer Rule. */ -public class LinkageCheckerRule extends AbstractNonCacheableEnforcerRule { +@Named("linkageCheckerRule") +public class LinkageCheckerRule extends AbstractEnforcerRule { /** * Maven packaging values known to be irrelevant to Linkage Check for non-BOM project. @@ -110,6 +110,31 @@ public class LinkageCheckerRule extends AbstractNonCacheableEnforcerRule { private ClassPathBuilder classPathBuilder; + // Properties managed by the dependency injection + private MavenProject project; + + private MavenSession session; + + private ProjectDependenciesResolver projectDependenciesResolver; + + private MojoExecution execution; + + private final RepositorySystem repoSystem; + + @Inject + public LinkageCheckerRule( + MavenProject project, + MavenSession session, + MojoExecution execution, + ProjectDependenciesResolver projectDependenciesResolver, + RepositorySystem repoSystem) { + this.project = project; + this.session = session; + this.execution = execution; + this.projectDependenciesResolver = projectDependenciesResolver; + this.repoSystem = repoSystem; + } + @VisibleForTesting void setDependencySection(DependencySection dependencySection) { this.dependencySection = dependencySection; @@ -133,139 +158,131 @@ void setExclusionFile(String exclusionFile) { this.exclusionFile = exclusionFile; } - private static Log logger; + private static EnforcerLogger logger; @Override - public void execute(@Nonnull EnforcerRuleHelper helper) throws EnforcerRuleException { - logger = helper.getLog(); + public void execute() throws EnforcerRuleException { + logger = getLog(); - try { - MavenProject project = (MavenProject) helper.evaluate("${project}"); - MavenSession session = (MavenSession) helper.evaluate("${session}"); - MojoExecution execution = (MojoExecution) helper.evaluate("${mojoExecution}"); - RepositorySystemSession repositorySystemSession = session.getRepositorySession(); - - ImmutableList repositoryUrls = - project.getRemoteProjectRepositories().stream() - .map(RemoteRepository::getUrl) - .collect(toImmutableList()); - DependencyGraphBuilder dependencyGraphBuilder = new DependencyGraphBuilder(repositoryUrls); - classPathBuilder = new ClassPathBuilder(dependencyGraphBuilder); - - boolean readingDependencyManagementSection = - dependencySection == DependencySection.DEPENDENCY_MANAGEMENT; - if (readingDependencyManagementSection - && (project.getDependencyManagement() == null - || project.getDependencyManagement().getDependencies() == null - || project.getDependencyManagement().getDependencies().isEmpty())) { - logger.warn("The rule is set to read dependency management section but it is empty."); - } + RepositorySystemSession repositorySystemSession = session.getRepositorySession(); - String projectType = project.getArtifact().getType(); - if (readingDependencyManagementSection) { - if (!"pom".equals(projectType)) { - logger.warn("A BOM should have packaging pom"); - return; - } - } else { - if (UNSUPPORTED_NONBOM_PACKAGING.contains(projectType)) { - return; - } - if (!"verify".equals(execution.getLifecyclePhase())) { - throw new EnforcerRuleException( - "To run the check on the compiled class files, the linkage checker enforcer rule" - + " should be bound to the 'verify' phase. Current phase: " - + execution.getLifecyclePhase()); - } - if (project.getArtifact().getFile() == null) { - // Skipping projects without a file, such as Guava's guava-tests module. - // https://github.com/GoogleCloudPlatform/cloud-opensource-java/issues/850 - return; - } - } + ImmutableList repositoryUrls = + project.getRemoteProjectRepositories().stream() + .map(RemoteRepository::getUrl) + .collect(toImmutableList()); + DependencyGraphBuilder dependencyGraphBuilder = new DependencyGraphBuilder(repositoryUrls); + classPathBuilder = new ClassPathBuilder(dependencyGraphBuilder); + + boolean readingDependencyManagementSection = + dependencySection == DependencySection.DEPENDENCY_MANAGEMENT; + if (readingDependencyManagementSection + && (project.getDependencyManagement() == null + || project.getDependencyManagement().getDependencies() == null + || project.getDependencyManagement().getDependencies().isEmpty())) { + logger.warn("The rule is set to read dependency management section but it is empty."); + } - ClassPathResult classPathResult = - readingDependencyManagementSection - ? findBomClasspath(project, repositorySystemSession) - : findProjectClasspath(project, repositorySystemSession, helper); - ImmutableList classPath = classPathResult.getClassPath(); - if (classPath.isEmpty()) { - logger.warn("Class path is empty."); + String projectType = project.getArtifact().getType(); + if (readingDependencyManagementSection) { + if (!"pom".equals(projectType)) { + logger.warn("A BOM should have packaging pom"); + return; + } + } else { + if (UNSUPPORTED_NONBOM_PACKAGING.contains(projectType)) { + return; + } + if (!"verify".equals(execution.getLifecyclePhase())) { + throw new EnforcerRuleException( + "To run the check on the compiled class files, the linkage checker enforcer rule" + + " should be bound to the 'verify' phase. Current phase: " + + execution.getLifecyclePhase()); + } + if (project.getArtifact().getFile() == null) { + // Skipping projects without a file, such as Guava's guava-tests module. + // https://github.com/GoogleCloudPlatform/cloud-opensource-java/issues/850 return; } + } - List entryPoints = entryPoints(project, classPath); - - try { - - // TODO LinkageChecker.create and LinkageChecker.findLinkageProblems - // should not be two separate public methods since we always call - // findLinkageProblems immediately after create. - - Path exclusionFile = this.exclusionFile == null ? null : Paths.get(this.exclusionFile); - LinkageChecker linkageChecker = - LinkageChecker.create(classPath, entryPoints, exclusionFile); - ImmutableSet linkageProblems = linkageChecker.findLinkageProblems(); - if (reportOnlyReachable) { - ClassReferenceGraph classReferenceGraph = linkageChecker.getClassReferenceGraph(); - linkageProblems = - linkageProblems.stream() - .filter( - entry -> - classReferenceGraph.isReachable(entry.getSourceClass().getBinaryName())) - .collect(toImmutableSet()); - } + ClassPathResult classPathResult = + readingDependencyManagementSection + ? findBomClasspath(project, repositorySystemSession) + : findProjectClasspath(project, repositorySystemSession, projectDependenciesResolver); + ImmutableList classPath = classPathResult.getClassPath(); + if (classPath.isEmpty()) { + logger.warn("Class path is empty."); + return; + } - if (classPathResult != null) { - LinkageProblemCauseAnnotator.annotate(classPathBuilder, classPathResult, linkageProblems); - } + List entryPoints = entryPoints(project, classPath); - // Count unique LinkageProblems by their symbols - long errorCount = - linkageProblems.stream().map(LinkageProblem::formatSymbolProblem).distinct().count(); + try { - String foundError = reportOnlyReachable ? "reachable error" : "error"; - if (errorCount > 1) { - foundError += "s"; - } - if (errorCount > 0) { - String message = - String.format( - "Linkage Checker rule found %d %s:\n%s", - errorCount, - foundError, - LinkageProblem.formatLinkageProblems(linkageProblems, classPathResult)); - if (getLevel() == WARN) { - logger.warn(message); - } else { - logger.error(message); - logger.info( - "For the details of the linkage errors, see " - + "https://github.com/GoogleCloudPlatform/cloud-opensource-java/wiki/Linkage-Checker-Messages"); - throw new EnforcerRuleException( - "Failed while checking class path. See above error report."); - } + // TODO LinkageChecker.create and LinkageChecker.findLinkageProblems + // should not be two separate public methods since we always call + // findLinkageProblems immediately after create. + + Path exclusionFile = this.exclusionFile == null ? null : Paths.get(this.exclusionFile); + LinkageChecker linkageChecker = LinkageChecker.create(classPath, entryPoints, ImmutableList.of(), exclusionFile); + ImmutableSet linkageProblems = linkageChecker.findLinkageProblems(); + if (reportOnlyReachable) { + ClassReferenceGraph classReferenceGraph = linkageChecker.getClassReferenceGraph(); + linkageProblems = + linkageProblems.stream() + .filter( + entry -> + classReferenceGraph.isReachable(entry.getSourceClass().getBinaryName())) + .collect(toImmutableSet()); + } + + if (classPathResult != null) { + LinkageProblemCauseAnnotator.annotate(classPathBuilder, classPathResult, linkageProblems); + } + + // Count unique LinkageProblems by their symbols + long errorCount = + linkageProblems.stream().map(LinkageProblem::formatSymbolProblem).distinct().count(); + + String foundError = reportOnlyReachable ? "reachable error" : "error"; + if (errorCount > 1) { + foundError += "s"; + } + if (errorCount > 0) { + String message = + String.format( + "Linkage Checker rule found %d %s:\n%s", + errorCount, + foundError, + LinkageProblem.formatLinkageProblems(linkageProblems, classPathResult)); + if (getLevel() == WARN) { + logger.warn(message); } else { - // arguably shouldn't log anything on success - logger.info("No " + foundError + " found"); + logger.error(message); + logger.info( + "For the details of the linkage errors, see " + + "https://github.com/GoogleCloudPlatform/cloud-opensource-java/wiki/Linkage-Checker-Messages"); + throw new EnforcerRuleException( + "Failed while checking class path. See above error report."); } - } catch (IOException ex) { - // Maven's "-e" flag does not work for EnforcerRuleException. Print stack trace here. - logger.warn("Failed to run Linkage Checker:" + ex.getMessage(), ex); - throw new EnforcerRuleException("Failed to run Linkage Checker", ex); + } else { + // arguably shouldn't log anything on success + logger.info("No " + foundError + " found"); } - } catch (ExpressionEvaluationException ex) { - throw new EnforcerRuleException("Unable to lookup an expression " + ex.getMessage(), ex); + } catch (IOException ex) { + // Maven's "-e" flag does not work for EnforcerRuleException + logger.warn("Failed to run Linkage Checker:" + ex); + throw new EnforcerRuleException("Failed to run Linkage Checker: " + ex, ex); } } /** Builds a class path for {@code mavenProject}. */ private static ClassPathResult findProjectClasspath( - MavenProject mavenProject, RepositorySystemSession session, EnforcerRuleHelper helper) + MavenProject mavenProject, + RepositorySystemSession session, + ProjectDependenciesResolver projectDependenciesResolver) throws EnforcerRuleException { try { - ProjectDependenciesResolver projectDependenciesResolver = - helper.getComponent(ProjectDependenciesResolver.class); DefaultRepositorySystemSession fullDependencyResolutionSession = new DefaultRepositorySystemSession(session); @@ -293,8 +310,6 @@ private static ClassPathResult findProjectClasspath( projectDependenciesResolver.resolve(dependencyResolutionRequest); return buildClassPathResult(resolutionResult); - } catch (ComponentLookupException e) { - throw new EnforcerRuleException("Unable to lookup a component " + e.getMessage(), e); } catch (DependencyResolutionException e) { return buildClasspathFromException(e); } diff --git a/enforcer-rules/src/test/java/com/google/cloud/tools/dependencies/enforcer/LinkageCheckerRuleTest.java b/enforcer-rules/src/test/java/com/google/cloud/tools/dependencies/enforcer/LinkageCheckerRuleTest.java index a6aec1f2de..7fd0dc427a 100644 --- a/enforcer-rules/src/test/java/com/google/cloud/tools/dependencies/enforcer/LinkageCheckerRuleTest.java +++ b/enforcer-rules/src/test/java/com/google/cloud/tools/dependencies/enforcer/LinkageCheckerRuleTest.java @@ -41,12 +41,11 @@ import java.util.stream.Collectors; import org.apache.maven.artifact.handler.DefaultArtifactHandler; import org.apache.maven.enforcer.rule.api.EnforcerLevel; +import org.apache.maven.enforcer.rule.api.EnforcerLogger; import org.apache.maven.enforcer.rule.api.EnforcerRuleException; -import org.apache.maven.enforcer.rule.api.EnforcerRuleHelper; import org.apache.maven.execution.MavenSession; import org.apache.maven.model.DependencyManagement; import org.apache.maven.plugin.MojoExecution; -import org.apache.maven.plugin.logging.Log; import org.apache.maven.project.DependencyResolutionException; import org.apache.maven.project.DependencyResolutionRequest; import org.apache.maven.project.DependencyResolutionResult; @@ -75,14 +74,13 @@ public class LinkageCheckerRuleTest { - private LinkageCheckerRule rule = new LinkageCheckerRule(); + private LinkageCheckerRule rule; private RepositorySystem repositorySystem; private RepositorySystemSession repositorySystemSession; private Artifact dummyArtifactWithFile; private MavenProject mockProject; - private EnforcerRuleHelper mockRuleHelper; - private Log mockLog; + private EnforcerLogger mockLog; private MavenSession mockMavenSession; private MojoExecution mockMojoExecution; private ProjectDependenciesResolver mockProjectDependenciesResolver; @@ -96,6 +94,15 @@ public void setup() repositorySystemSession = RepositoryUtility.newSession(repositorySystem); dummyArtifactWithFile = createArtifactWithDummyFile("a:b:0.1"); setupMock(); + + rule = + new LinkageCheckerRule( + mockProject, + mockMavenSession, + mockMojoExecution, + mockProjectDependenciesResolver, + repositorySystem); + rule.setLog(mockLog); } private Artifact createArtifactWithDummyFile(String coordinates) throws URISyntaxException { @@ -109,20 +116,13 @@ private void setupMock() mockProject = mock(MavenProject.class); mockMavenSession = mock(MavenSession.class); when(mockMavenSession.getRepositorySession()).thenReturn(repositorySystemSession); - mockRuleHelper = mock(EnforcerRuleHelper.class); mockProjectDependenciesResolver = mock(ProjectDependenciesResolver.class); mockDependencyResolutionResult = mock(DependencyResolutionResult.class); - mockLog = mock(Log.class); - when(mockRuleHelper.getLog()).thenReturn(mockLog); - when(mockRuleHelper.getComponent(ProjectDependenciesResolver.class)) - .thenReturn(mockProjectDependenciesResolver); + mockLog = mock(EnforcerLogger.class); when(mockProjectDependenciesResolver.resolve(any(DependencyResolutionRequest.class))) .thenReturn(mockDependencyResolutionResult); - when(mockRuleHelper.evaluate("${session}")).thenReturn(mockMavenSession); - when(mockRuleHelper.evaluate("${project}")).thenReturn(mockProject); mockMojoExecution = mock(MojoExecution.class); when(mockMojoExecution.getLifecyclePhase()).thenReturn("verify"); - when(mockRuleHelper.evaluate("${mojoExecution}")).thenReturn(mockMojoExecution); org.apache.maven.artifact.DefaultArtifact rootArtifact = new org.apache.maven.artifact.DefaultArtifact( "com.google.cloud", @@ -194,7 +194,7 @@ public void testExecute_shouldPassGoodProject() // Since Guava 27, it requires com.google.guava:failureaccess artifact in its dependency. setupMockDependencyResolution("com.google.guava:guava:27.0.1-jre"); // This should not raise an EnforcerRuleException - rule.execute(mockRuleHelper); + rule.execute(); verify(mockLog).info("No error found"); } @@ -203,7 +203,7 @@ public void testExecute_shouldPassGoodProject_sessionProperties() throws EnforcerRuleException, RepositoryException, DependencyResolutionException { setupMockDependencyResolution("com.google.guava:guava:27.0.1-jre"); - rule.execute(mockRuleHelper); + rule.execute(); ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(DependencyResolutionRequest.class); @@ -225,7 +225,7 @@ public void testExecute_shouldFailForBadProject() throws RepositoryException { try { // This artifact is known to contain classes missing dependencies setupMockDependencyResolution("com.google.appengine:appengine-api-1.0-sdk:1.9.64"); - rule.execute(mockRuleHelper); + rule.execute(); Assert.fail( "The rule should raise an EnforcerRuleException for artifacts missing dependencies"); } catch (EnforcerRuleException ex) { @@ -259,7 +259,7 @@ public void testExecute_shouldFailForBadProject_reachableErrors() throws Reposit setupMockDependencyResolution( "com.google.api-client:google-api-client:1.27.0", "io.grpc:grpc-core:1.17.1"); rule.setReportOnlyReachable(true); - rule.execute(mockRuleHelper); + rule.execute(); Assert.fail( "The rule should raise an EnforcerRuleException for artifacts with reachable errors"); } catch (EnforcerRuleException ex) { @@ -278,9 +278,23 @@ public void testExecute_shouldPassForBadProject_levelWarn() // grpc-core is included in entry point jars, the errors are reachable. setupMockDependencyResolution( "com.google.api-client:google-api-client:1.27.0", "io.grpc:grpc-core:1.17.1"); + + rule = + new LinkageCheckerRule( + mockProject, + mockMavenSession, + mockMojoExecution, + mockProjectDependenciesResolver, + repositorySystem) { + @Override + public EnforcerLevel getLevel() { + return EnforcerLevel.WARN; + } + }; + rule.setLog(mockLog); + rule.setReportOnlyReachable(true); - rule.setLevel(EnforcerLevel.WARN); - rule.execute(mockRuleHelper); + rule.execute(); verify(mockLog) .warn(ArgumentMatchers.startsWith("Linkage Checker rule found 1 reachable error:")); } @@ -293,7 +307,7 @@ public void testExecute_shouldPassGoodProject_unreachableErrors() setupMockDependencyResolution("com.google.cloud:google-cloud-automl:0.81.0-beta"); rule.setReportOnlyReachable(true); // This should not raise EnforcerRuleException because the linkage errors are unreachable. - rule.execute(mockRuleHelper); + rule.execute(); } private void setupMockDependencyManagementSection(String... coordinates) { @@ -341,7 +355,7 @@ public void testExecute_shouldPassEmptyBom() throws EnforcerRuleException { setupMockDependencyManagementSection(); // empty BOM // This should not raise an EnforcerRuleException - rule.execute(mockRuleHelper); + rule.execute(); } @Test @@ -352,7 +366,7 @@ public void testExecute_shouldPassGoodBom() throws EnforcerRuleException { "io.grpc:grpc-auth:1.18.0", "com.google.api:api-common:1.7.0"); // This should not raise an EnforcerRuleException - rule.execute(mockRuleHelper); + rule.execute(); } @Test @@ -362,7 +376,7 @@ public void testExecute_shouldFailBadBom() { "com.google.api-client:google-api-client:1.27.0", "io.grpc:grpc-core:1.17.1"); try { - rule.execute(mockRuleHelper); + rule.execute(); Assert.fail("Enforcer rule should detect conflict between google-api-client and grpc-core"); } catch (EnforcerRuleException ex) { // pass @@ -384,7 +398,7 @@ public void testExecute_shouldSkipBadBomWithNonPomPackaging() throws EnforcerRul "jar", // BOM should have pom here null, new DefaultArtifactHandler())); - rule.execute(mockRuleHelper); + rule.execute(); } @Test @@ -400,7 +414,7 @@ public void testExecute_shouldSkipNonBomPom() throws EnforcerRuleException { null, new DefaultArtifactHandler())); // No exception - rule.execute(mockRuleHelper); + rule.execute(); } @Test @@ -419,7 +433,7 @@ public void testExecute_shouldExcludeTestScope() throws EnforcerRuleException { when(mockProject.getDependencies()) .thenReturn(ImmutableList.of(dependency)); - rule.execute(mockRuleHelper); + rule.execute(); } @Test @@ -440,7 +454,7 @@ public void testExecute_shouldFailForBadProjectWithBundlePackaging() throws Repo rootArtifact.setFile(new File("dummy.jar")); when(mockProject.getArtifact()).thenReturn(rootArtifact); - rule.execute(mockRuleHelper); + rule.execute(); Assert.fail( "The rule should raise an EnforcerRuleException for artifacts missing dependencies"); } catch (EnforcerRuleException ex) { @@ -470,7 +484,7 @@ public void testExecute_shouldFilterExclusionRule_java8() .toAbsolutePath() .toString(); rule.setExclusionFile(exclusionFileLocation); - rule.execute(mockRuleHelper); + rule.execute(); Assert.fail( "The rule should raise an EnforcerRuleException for artifacts missing dependencies"); } catch (EnforcerRuleException ex) { @@ -510,7 +524,7 @@ public void testArtifactTransferError() when(mockProjectDependenciesResolver.resolve(any())).thenThrow(exception); try { - rule.execute(mockRuleHelper); + rule.execute(); fail("The rule should throw EnforcerRuleException upon dependency resolution exception"); } catch (EnforcerRuleException expected) { verify(mockLog) @@ -565,7 +579,7 @@ public void testArtifactTransferError_acceptableMissingArtifact() // Should not throw DependencyResolutionException, because the missing xerces-impl is under both // provided and optional dependencies. - rule.execute(mockRuleHelper); + rule.execute(); } @Test @@ -603,7 +617,7 @@ public void testArtifactTransferError_missingArtifactNotInGraph() when(mockProjectDependenciesResolver.resolve(any())).thenThrow(exception); - rule.execute(mockRuleHelper); + rule.execute(); verify(mockLog) .warn("xerces:xerces-impl:jar:2.6.2 was not resolved. Dependency path is unknown."); } @@ -620,7 +634,7 @@ public void testSkippingProjectWithoutFile() throws EnforcerRuleException { "jar", null, new DefaultArtifactHandler())); - rule.execute(mockRuleHelper); + rule.execute(); } @Test @@ -638,7 +652,7 @@ public void testValidatePhase() { when(mockMojoExecution.getLifecyclePhase()).thenReturn("validate"); try { - rule.execute(mockRuleHelper); + rule.execute(); fail("The rule should throw EnforcerRuleException when running in validate phase"); } catch (EnforcerRuleException ex) { assertEquals( diff --git a/gradle-plugin/build.gradle b/gradle-plugin/build.gradle index cac6808473..d921ff18da 100644 --- a/gradle-plugin/build.gradle +++ b/gradle-plugin/build.gradle @@ -27,6 +27,8 @@ group = 'com.google.cloud.tools' sourceCompatibility = 1.8 +version = '1.5.16-SNAPSHOT' // {x-version-update:dependencies:current} + dependencies { implementation "com.google.cloud.tools:dependencies:$version" implementation 'com.google.guava:guava:30.1-jre' diff --git a/gradle-plugin/gradle.properties b/gradle-plugin/gradle.properties deleted file mode 100644 index e89ec876a4..0000000000 --- a/gradle-plugin/gradle.properties +++ /dev/null @@ -1,2 +0,0 @@ -# scripts/prepare_release.sh maintains this value. -version = 1.5.12-SNAPSHOT diff --git a/gradle-plugin/kokoro/release.sh b/gradle-plugin/kokoro/release.sh index 3c6632cb80..8f3e27f428 100755 --- a/gradle-plugin/kokoro/release.sh +++ b/gradle-plugin/kokoro/release.sh @@ -18,7 +18,7 @@ cat "${KOKORO_KEYSTORE_DIR}/72743_gradle_publish_secret" >> $HOME_GRADLE_PROPERT # The gradle plugin depends on the dependencies module cd github/cloud-opensource-java -./mvnw -V -B clean install +./mvnw -V -B -ntp clean install cd gradle-plugin diff --git a/gradle-plugin/src/functionalTest/groovy/com/google/cloud/tools/dependencies/gradle/BuildStatusFunctionalTest.groovy b/gradle-plugin/src/functionalTest/groovy/com/google/cloud/tools/dependencies/gradle/BuildStatusFunctionalTest.groovy index b9e7d05991..05d5bb43c4 100644 --- a/gradle-plugin/src/functionalTest/groovy/com/google/cloud/tools/dependencies/gradle/BuildStatusFunctionalTest.groovy +++ b/gradle-plugin/src/functionalTest/groovy/com/google/cloud/tools/dependencies/gradle/BuildStatusFunctionalTest.groovy @@ -224,4 +224,38 @@ class BuildStatusFunctionalTest extends Specification { result.output.count("Circular dependency for: com.fasterxml.jackson:jackson-bom:2.12.1") == 1 result.task(":linkageCheck").outcome == TaskOutcome.FAILED } + + def "can handle artifacts with pom packaging"() { + buildFile << """ + repositories { + mavenCentral() + } + + dependencies { + // This artifact is pom-packaging and does not have JAR artifacts + compile 'org.eclipse.jetty.aggregate:jetty-all:9.4.7.v20170914' + } + + linkageChecker { + configurations = ['compile'] + } + """ + + when: + def result = GradleRunner.create() + .withProjectDir(testProjectDir.root) + .withArguments('linkageCheck', '--stacktrace') + .withPluginClasspath() + .buildAndFail() + + then: + result.output.contains("Task :linkageCheck") + result.output.contains( + "The valid symbol is in org.eclipse.jetty.alpn:alpn-api:jar:1.1.3.v20160715 at "+ + "org.eclipse.jetty.aggregate:jetty-all:9.4.7.v20170914 (compile) / "+ + "org.eclipse.jetty:jetty-alpn-client:9.4.7.v20170914 (compile) / "+ + "org.eclipse.jetty.alpn:alpn-api:1.1.3.v20160715 (provided) but it was not selected "+ + "because the path contains a provided-scope dependency") + result.task(":linkageCheck").outcome == TaskOutcome.FAILED + } } diff --git a/gradle-plugin/src/main/java/com/google/cloud/tools/dependencies/gradle/LinkageCheckTask.java b/gradle-plugin/src/main/java/com/google/cloud/tools/dependencies/gradle/LinkageCheckTask.java index b4dc7d96c0..1526af1edf 100644 --- a/gradle-plugin/src/main/java/com/google/cloud/tools/dependencies/gradle/LinkageCheckTask.java +++ b/gradle-plugin/src/main/java/com/google/cloud/tools/dependencies/gradle/LinkageCheckTask.java @@ -34,6 +34,7 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.ListMultimap; import com.google.common.collect.MultimapBuilder; +import java.io.File; import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; @@ -144,7 +145,7 @@ private boolean findLinkageErrors(Configuration configuration) throws IOExceptio } // TODO(suztomo): Specify correct entry points if reportOnlyReachable is true. - LinkageChecker linkageChecker = LinkageChecker.create(classPath, classPath, exclusionFile); + LinkageChecker linkageChecker = LinkageChecker.create(classPath, classPath, ImmutableList.of(), exclusionFile); ImmutableSet linkageProblems = linkageChecker.findLinkageProblems(); @@ -314,6 +315,20 @@ private String dependencyPathToArtifacts( return output.toString(); } + private static Artifact artifactWithPomFrom(ResolvedDependency resolvedDependency) { + ModuleVersionIdentifier moduleVersionId = resolvedDependency.getModule().getId(); + DefaultArtifact artifact = + new DefaultArtifact( + moduleVersionId.getGroup(), + moduleVersionId.getName(), + null, + "pom", + moduleVersionId.getVersion(), + null, + (File) null); // no JAR file + return artifact; + } + private static Artifact artifactFrom( ResolvedDependency resolvedDependency, ResolvedArtifact resolvedArtifact) { ModuleVersionIdentifier moduleVersionId = resolvedDependency.getModule().getId(); @@ -357,6 +372,7 @@ private DependencyGraph createDependencyGraph(ResolvedConfiguration configuratio PathToNode item = queue.poll(); ResolvedDependency node = item.getNode(); + // Never null DependencyPath parentPath = item.getParentPath(); // If there are multiple artifacts (with different classifiers) in this node, then the path is @@ -372,18 +388,15 @@ private DependencyGraph createDependencyGraph(ResolvedConfiguration configuratio getLogger() .warn( "The dependency node " + node.getName() + " does not have any artifact. Skipping."); - continue; + path = parentPath.append(new Dependency(artifactWithPomFrom(node), "compile")); + } else { + // For artifacts with classifiers, there can be multiple resolved artifacts for one node + for (ResolvedArtifact artifact : moduleArtifacts) { + path = parentPath.append(dependencyFrom(node, artifact)); + graph.addPath(path); + } } - // For artifacts with classifiers, there can be multiple resolved artifacts for one node - for (ResolvedArtifact artifact : moduleArtifacts) { - // parentPath is null for the first item - path = - parentPath == null - ? new DependencyPath(artifactFrom(node, artifact)) - : parentPath.append(dependencyFrom(node, artifact)); - graph.addPath(path); - } for (ResolvedDependency child : node.getChildren()) { if (visited.add(child)) { diff --git a/kokoro/continuous.bat b/kokoro/continuous.bat index 0d758ee8c3..df44fdaae8 100644 --- a/kokoro/continuous.bat +++ b/kokoro/continuous.bat @@ -1,11 +1,12 @@ @echo on -set JAVA_HOME=c:\program files\java\jdk1.8.0_152 +set JAVA_HOME=c:\program files\java\jdk1.8.0_211 set PATH=%JAVA_HOME%\bin;%PATH% +set MAVEN_OPTS="-Xmx12g" cd github/cloud-opensource-java -call mvnw.cmd -V -B clean install javadoc:jar +call mvnw.cmd -V -B -ntp clean install javadoc:jar if %errorlevel% neq 0 exit /b %errorlevel% @echo on diff --git a/kokoro/continuous.sh b/kokoro/continuous.sh index 86ab10066f..374f9e7941 100755 --- a/kokoro/continuous.sh +++ b/kokoro/continuous.sh @@ -5,9 +5,11 @@ set -e # Display commands being run. set -x +export MAVEN_OPTS="-Xmx8g" + cd github/cloud-opensource-java -./mvnw -V -B -ntp clean install javadoc:jar +./mvnw -V -B -X -ntp clean install -Djavadoc.skip cd gradle-plugin ./gradlew build publishToMavenLocal diff --git a/kokoro/macos_external/continuous.cfg b/kokoro/macos_external/continuous.cfg deleted file mode 100644 index 43ded92399..0000000000 --- a/kokoro/macos_external/continuous.cfg +++ /dev/null @@ -1,4 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -# Location of the continuous build bash script in git. -build_file: "cloud-opensource-java/kokoro/continuous.sh" diff --git a/kokoro/macos_external/presubmit.cfg b/kokoro/macos_external/presubmit.cfg deleted file mode 100644 index 7cbe5d0058..0000000000 --- a/kokoro/macos_external/presubmit.cfg +++ /dev/null @@ -1,4 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -# Location of the presubmit build bash script in git. -build_file: "cloud-opensource-java/kokoro/continuous.sh" diff --git a/kokoro/parent_release_build.sh b/kokoro/parent_release_build.sh index de48e414cd..9aebcb0925 100755 --- a/kokoro/parent_release_build.sh +++ b/kokoro/parent_release_build.sh @@ -9,7 +9,7 @@ cd github/cloud-opensource-java # Build the project (excluding boms) to ensure validity of parent pom # The artifact is unused in this parent-pom build. -./mvnw -V -pl 'dependencies,enforcer-rules' -Prelease -B -U verify +./mvnw -V -pl 'dependencies,enforcer-rules' -Prelease -B -ntp -U verify # copy pom with the name expected in the Maven repository ARTIFACT_ID=$(mvn -B help:evaluate -Dexpression=project.artifactId 2>/dev/null | grep -v "^\[") diff --git a/kokoro/ubuntu/java8-incompatible-reference-check.sh b/kokoro/ubuntu/java8-incompatible-reference-check.sh deleted file mode 100644 index f2bd70480c..0000000000 --- a/kokoro/ubuntu/java8-incompatible-reference-check.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -# Fail on any error. -set -e -# Display commands being run. -set -x - -cd github/cloud-opensource-java/dependencies -mvn -V -B clean install - -mvn exec:java -e -Pjava8-incompatible-reference-check \ No newline at end of file diff --git a/kokoro/ubuntu/periodic.cfg b/kokoro/ubuntu/periodic.cfg deleted file mode 100644 index 68610342db..0000000000 --- a/kokoro/ubuntu/periodic.cfg +++ /dev/null @@ -1,11 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -# Location of the periodic build bash script in git. -build_file: "cloud-opensource-java/kokoro/ubuntu/periodic.sh" - -action { - define_artifacts { - regex: "**/target/com.google.cloud/**" - strip_prefix: "github/cloud-opensource-java/dashboard/target" - } -} diff --git a/kokoro/ubuntu/periodic.sh b/kokoro/ubuntu/periodic.sh deleted file mode 100755 index 152aafb3c0..0000000000 --- a/kokoro/ubuntu/periodic.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -# Fail on any error. -set -e -# Display commands being run. -set -x - -cd github/cloud-opensource-java -# M2_HOME is not used since Maven 3.5.0 https://maven.apache.org/docs/3.5.0/release-notes.html -mvn -V -B clean install - -# Running target of dashboard submodule -# https://stackoverflow.com/questions/3459928/running-a-specific-maven-plugin-goal-from-the-command-line-in-a-sub-module-of-a/26448447#26448447 -# https://stackoverflow.com/questions/11091311/maven-execjava-goal-on-a-multi-module-project -cd dashboard - -# For all versions available in Maven Central and local repository -mvn -V -B exec:java -Dexec.mainClass="com.google.cloud.tools.opensource.dashboard.DashboardMain" \ - -Dexec.arguments="-a com.google.cloud:libraries-bom" - -mvn -V -B exec:java -Dexec.mainClass="com.google.cloud.tools.opensource.dashboard.DashboardMain" \ - -Dexec.arguments="-a com.google.cloud:gcp-lts-bom" diff --git a/linkage-monitor/action.yml b/linkage-monitor/action.yml index 69de4893ba..de4b76d898 100644 --- a/linkage-monitor/action.yml +++ b/linkage-monitor/action.yml @@ -7,7 +7,7 @@ runs: # scripts/release.sh updates the version part in the URL run: | curl --output /tmp/linkage-monitor.jar \ - "https://storage.googleapis.com/cloud-opensource-java-linkage-monitor/linkage-monitor-1.5.12-SNAPSHOT-all-deps.jar" + "https://storage.googleapis.com/cloud-opensource-java-linkage-monitor/linkage-monitor-1.5.16-SNAPSHOT-all-deps.jar" shell: bash - run: java -jar /tmp/linkage-monitor.jar com.google.cloud:libraries-bom shell: bash diff --git a/linkage-monitor/pom.xml b/linkage-monitor/pom.xml index b04716f465..229975e4a5 100644 --- a/linkage-monitor/pom.xml +++ b/linkage-monitor/pom.xml @@ -22,7 +22,7 @@ com.google.cloud.tools dependencies-parent - 1.5.12-SNAPSHOT + 1.5.16-SNAPSHOT linkage-monitor diff --git a/linkage-monitor/src/main/java/com/google/cloud/tools/dependencies/linkagemonitor/LinkageMonitor.java b/linkage-monitor/src/main/java/com/google/cloud/tools/dependencies/linkagemonitor/LinkageMonitor.java index 9e492a83d4..42cff216dd 100644 --- a/linkage-monitor/src/main/java/com/google/cloud/tools/dependencies/linkagemonitor/LinkageMonitor.java +++ b/linkage-monitor/src/main/java/com/google/cloud/tools/dependencies/linkagemonitor/LinkageMonitor.java @@ -294,7 +294,7 @@ private ImmutableSet run(String groupId, String artifactId) List entryPointJars = classpath.subList(0, snapshotManagedDependencies.size()); ImmutableSet problemsInSnapshot = - LinkageChecker.create(classpath, ImmutableSet.copyOf(entryPointJars), null) + LinkageChecker.create(classpath, ImmutableSet.copyOf(entryPointJars), ImmutableList.of(), null) .findLinkageProblems(); if (problemsInBaseline.equals(problemsInSnapshot)) { diff --git a/pom.xml b/pom.xml index aaf9e176af..215bbe8486 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ com.google.cloud.tools dependencies-parent pom - 1.5.12-SNAPSHOT + 1.5.16-SNAPSHOT Cloud Tools Open Source Code Hygiene Tooling https://github.com/GoogleCloudPlatform/cloud-opensource-java/ @@ -41,7 +41,7 @@ UTF-8 30.1.1-jre 9+181-r4173-1 - 3.6.3 + 3.8.8 1.7.1 1.1.3 @@ -54,11 +54,8 @@ - boms dependencies - dashboard enforcer-rules - linkage-monitor @@ -249,6 +246,11 @@ release + + + performRelease + + @@ -277,6 +279,62 @@ + + org.apache.maven.plugins + maven-gpg-plugin + 3.2.7 + + + sign-artifacts + verify + + sign + + + + --pinentry-mode + loopback + + + + + + + + + + codecoverage + + + + maven-surefire-plugin + + + @{argLine} -Xms1024m -Xmx1024m + + **/*Test.java + + + + + org.jacoco + jacoco-maven-plugin + 0.8.6 + + + + prepare-agent + + + + report + test + + report + + + + diff --git a/release-please-config.json b/release-please-config.json new file mode 100644 index 0000000000..340b223ff8 --- /dev/null +++ b/release-please-config.json @@ -0,0 +1,17 @@ +{ + "$schema": "https://raw.githubusercontent.com/googleapis/release-please/main/schemas/config.json", + "separate-pull-requests": true, + "include-component-in-tag": true, + "packages": { + "dependencies": { + "component": "dependencies", + "release-type": "java-yoshi", + "extra-files": ["pom.xml", "enforcer-rules/pom.xml", "gradle-plugin/build.gradle", "linkage-monitor/pom.xml"] + }, + "boms/cloud-lts-bom": { + "component": "gcp-lts-bom", + "release-type": "java-yoshi" + } + }, + "bootstrap-sha": "6b9240114536a03b72929d5fade85599fbdbbdd0" +} diff --git a/scripts/release.sh b/scripts/release.sh index 7de44d7c6a..80b73d7f5c 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -124,8 +124,8 @@ git push origin "${RELEASE_TAG}" git push --set-upstream origin ${VERSION}-${SUFFIX} # Create the PR -gh pr create --title "Release ${VERSION}-${SUFFIX}" \ - --body "Release ${VERSION}-${SUFFIX}" \ +gh pr create --title "Release ${VERSION}-${SUFFIX}: Bumping to next version post release" \ + --body "1st commit marks ${VERSION}-${SUFFIX} release with the Git tag. 2nd commit bumps the version in the branch with SNAPSHOT suffix." \ --base ${BASE_BRANCH} # File a PR on Github for the new branch. Have someone LGTM it, which gives you permission to continue. diff --git a/temurin-install-testing/.gitignore b/temurin-install-testing/.gitignore new file mode 100644 index 0000000000..587889aad2 --- /dev/null +++ b/temurin-install-testing/.gitignore @@ -0,0 +1,7 @@ +*.tfstate.backup +*.tfstate +private.auto.tfvars +.terraform/ +.idea/ +.terraform.lock.hcl +.terraform.tfstate.lock.info diff --git a/temurin-install-testing/README.md b/temurin-install-testing/README.md new file mode 100644 index 0000000000..8bb6580faa --- /dev/null +++ b/temurin-install-testing/README.md @@ -0,0 +1,62 @@ +# temurin-testing + +This Terraform configuration installs Temurin on a matrix of GCE machine types and OS boot images. +The tests are split into three batches to prevent hitting standard compute quotas, and are enabled +using three boolean input variables: + +* `enable_arm`: Performs testing on `arm_machine_types` x `arm_boot_images` +* `enable_linux`: Performs testing on `x86_machine_types` x `x86_boot_images` +* `enable_windows`: Performs testing on `x86_machine_types` x `x86_windows_boot_images` + +Defaults for the above values are found in `variables.tf`. + +## Snippet Region Tags + +The `startup.sh` and `startup.ps1` files have region tags used to identify the boundaries of the +snippets to include in Temurin installation documentation. + +## Workflow + +1. Create a file named `private.auto.tfvars` in this folder with contents: + ```shell + user_email = "[gcloud account email]" + project_id = "[gcp project id]" + bucket = "[cloud storage bucket id for results]" + ``` + +2. Invoke the tests + * To invoke the full test suite, execute `terraform init`, then `./test-all.sh` + * To invoke a single test batch, modify `inputs.auto.tfvars` to enable one batch, then execute + `terraform apply`. To destroy these resources later, set the batch variable back + to false and execute `terraform apply` again.cd + * (Optional) Review the input matrix for the batch you've enabled in `variables.tf`. + +3. (Parsing Results) Test results are uploaded to the specified GCS bucket at + `gs://[BUCKET]/[TIMESTAMP]/[MACHINE TYPE]` and full logs are available + at `gs://[BUCKET]/[TIMESTAMP]/logs`. + +4. (Debugging) To SSH into a created VM instance: + * Find the VM instance name by either navigating to + https://console.cloud.google.com/compute/instances, or by looking at Terraform's console + output. + * Example Terraform output: + ``` + google_compute_instance.default["n2-standard-2-centos-stream-8"]: Creation complete after 14s [id=projects/PROJECT_ID/zones/ZONE_ID/instances/n2-standard-2-centos-stream-8] + ``` + Where `n2-standard-2-centos-stream-8` is the VM instance name. + * Invoke the `connect.sh` helper script with the VM instance name: + ```shell + ./connect.sh [VM_INSTANCE_NAME] + ``` + Example: + ```shell + ./connect.sh n2-standard-2-centos-stream-8 + ``` + +## Support + +This repository is not intended for public consumption. + +For source code and support for Google cloud libraries, start here: + +https://cloud.google.com/apis/docs/cloud-client-libraries \ No newline at end of file diff --git a/temurin-install-testing/connect.sh b/temurin-install-testing/connect.sh new file mode 100755 index 0000000000..f1ebfb77f0 --- /dev/null +++ b/temurin-install-testing/connect.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +# +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -eo pipefail + +ZONE=$(terraform output -raw zone) +PROJECT=$(terraform output -raw project) + +echo "" +echo "Once connection is complete, to see startup script logs in Linux:" +echo " sudo journalctl -u google-startup-scripts.service" +echo "To rerun startup script in Windows:" +echo ' "C:\Program Files\Google\Compute Engine\metadata_scripts\run_startup_scripts.cmd"' + +gcloud compute ssh --project="$PROJECT" --zone="$ZONE" "$1" diff --git a/temurin-install-testing/inputs.auto.tfvars b/temurin-install-testing/inputs.auto.tfvars new file mode 100644 index 0000000000..bee7d57c4c --- /dev/null +++ b/temurin-install-testing/inputs.auto.tfvars @@ -0,0 +1,4 @@ +# Only enable one test category at a time to avoid hitting default compute quotas. +enable_arm = false +enable_windows = false +enable_linux = false diff --git a/temurin-install-testing/main.tf b/temurin-install-testing/main.tf new file mode 100644 index 0000000000..2f2090ca03 --- /dev/null +++ b/temurin-install-testing/main.tf @@ -0,0 +1,176 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + } +} +provider "google" { + project = var.project_id + region = var.region + zone = var.zone +} +resource "google_project_service" "compute" { + service = "compute.googleapis.com" + disable_on_destroy = false +} +resource "google_project_service" "storage" { + service = "storage.googleapis.com" + disable_on_destroy = false +} + +resource "google_service_account" "default" { + account_id = "temurin-service-account" + display_name = "Service Account" +} + + +locals { + x86_instances = var.enable_linux ? setproduct(var.x86_machine_types, var.x86_boot_images) : [] + arm_instances = var.enable_arm ? setproduct(var.arm_machine_types, var.arm_boot_images) : [] + + linux_instances = [ + for entry in concat(local.x86_instances, local.arm_instances) : { + name = "${entry[0]}-${split("/", entry[1])[1]}" + type = entry[0] + image = entry[1] + os_name = split("/", entry[1])[1] + } + ] + + windows_instances = var.enable_windows ? [ + for entry in setproduct(var.x86_machine_types, var.x86_windows_boot_images) : { + name = "${entry[0]}-${split("/", entry[1])[1]}" + type = entry[0] + image = entry[1] + os_name = split("/", entry[1])[1] + } + ] : [] + + all_instances = concat(local.linux_instances, local.windows_instances) + + bucket_folder = var.bucket_folder == "" ? timestamp() : var.bucket_folder +} + +resource "google_compute_instance" "windows" { + for_each = { + for index, vm in local.windows_instances : vm.name => vm + } + + name = each.value.name + machine_type = each.value.type + tags = ["https-server", "http-server"] + + metadata = { + windows-startup-script-cmd = "googet -noconfirm=true update && googet -noconfirm=true install google-compute-engine-ssh" + enable-windows-ssh = "true" + serial-port-logging-enable = "true" + windows-startup-script-ps1 = templatefile( + "${path.module}/startup.ps1", + { + bucket = data.google_storage_bucket.results.name + vm_name = each.value.name + bucket_folder = local.bucket_folder + vm_zone = var.zone + os_name = each.value.os_name + machine_type = each.value.type + }) + } + + boot_disk { + initialize_params { + image = each.value.image + } + } + network_interface { + network = "default" + access_config { + // Ephemeral public IP + } + } + service_account { + email = google_service_account.default.email + scopes = ["cloud-platform"] + } + + depends_on = [ + google_project_service.compute, + google_service_account.default, + google_storage_bucket_iam_policy.storage_policy + ] +} + +resource "google_compute_instance" "linux" { + for_each = { + for index, vm in local.linux_instances : vm.name => vm + } + + name = each.value.name + machine_type = each.value.type + tags = ["https-server", "http-server"] + metadata_startup_script = templatefile( + "${path.module}/startup.sh", + { + bucket = data.google_storage_bucket.results.name + vm_name = each.value.name + bucket_folder = local.bucket_folder + vm_zone = var.zone + os_name = each.value.os_name + machine_type = each.value.type + }) + + boot_disk { + initialize_params { + image = each.value.image + } + } + network_interface { + network = "default" + access_config { + // Ephemeral public IP + } + } + service_account { + email = google_service_account.default.email + scopes = ["cloud-platform"] + } + + depends_on = [ + google_project_service.compute, + google_service_account.default, + google_storage_bucket_iam_policy.storage_policy + ] +} + +data "google_iam_policy" "compute_viewer" { + binding { + role = "roles/compute.viewer" + members = [ + "serviceAccount:${google_service_account.default.email}" + ] + } +} +resource "google_compute_instance_iam_policy" "compute_policy" { + for_each = { + for index, vm in local.all_instances : vm.name => vm + } + instance_name = each.value.name + policy_data = data.google_iam_policy.compute_viewer.policy_data + depends_on = [google_compute_instance.linux, google_compute_instance.windows] +} + +data "google_storage_bucket" "results" { + name = var.bucket +} +data "google_iam_policy" "storage_policy" { + binding { + role = "roles/storage.admin" + members = [ + "serviceAccount:${google_service_account.default.email}" + ] + } +} +resource "google_storage_bucket_iam_policy" "storage_policy" { + bucket = data.google_storage_bucket.results.name + policy_data = data.google_iam_policy.storage_policy.policy_data +} diff --git a/temurin-install-testing/outputs.tf b/temurin-install-testing/outputs.tf new file mode 100644 index 0000000000..d8d198314e --- /dev/null +++ b/temurin-install-testing/outputs.tf @@ -0,0 +1,9 @@ +output "zone" { + value = var.zone +} +output "project" { + value = var.project_id +} +output "bucket_folder" { + value = local.bucket_folder +} diff --git a/temurin-install-testing/startup.ps1 b/temurin-install-testing/startup.ps1 new file mode 100644 index 0000000000..168274bcd9 --- /dev/null +++ b/temurin-install-testing/startup.ps1 @@ -0,0 +1,96 @@ +# +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +function Refresh-Path +{ + # [START windows_env_reset] + $MachinePath = [System.Environment]::GetEnvironmentVariable('Path', 'Machine') + $UserPath = [System.Environment]::GetEnvironmentVariable('Path', 'User') + $env:Path = "$MachinePath;$UserPath" + # [END windows_env_reset] +} + +function Perform-Test +{ + param ( + $JdkVersion + ) + $BaseFileName = "${os_name}-jdk$JdkVersion" + $SuccessFileName = "$BaseFileName.txt" + $ErrorFileName = "$BaseFileName-error.txt" + $OriginalPath = $env:Path + + try + { + # [START windows_temurin_download] + $JdkUrl = "https://api.adoptium.net/v3/binary/latest/$JdkVersion/ga/windows/x64/jdk/hotspot/normal/eclipse?project=jdk" + $JdkExtractionPath = "C:\temurin-$JdkVersion-jdk" + $JdkDownload = "$JdkExtractionPath.zip" + [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.SecurityProtocolType]'Tls12' + Invoke-WebRequest -Uri $JdkUrl -OutFile $JdkDownload + Expand-Archive $JdkDownload -DestinationPath $JdkExtractionPath -Force + # [END windows_temurin_download] + "Downloaded: $JdkUrl" + + # [START windows_temurin_install] + pushd $JdkExtractionPath + $JdkPath = (Get-ChildItem).FullName + popd + [System.Environment]::SetEnvironmentVariable('JAVA_HOME', $JdkPath, 'Machine') + [System.Environment]::SetEnvironmentVariable('Path', "$env:Path;$JdkPath\bin", 'Machine') + # [END windows_temurin_install] + + Refresh-Path + java -version 2>&1 | %%{ "$_" } > $SuccessFileName # For syntax, see https://stackoverflow.com/a/20950421 + + # Reset for next test + [System.Environment]::SetEnvironmentVariable('JAVA_HOME', '', 'Machine') + [System.Environment]::SetEnvironmentVariable('Path', $OriginalPath, 'Machine') + Refresh-Path + try + { + java -version # Expect failure. + + "Java not fully uninstalled from Path: $env:Path" | Out-File -FilePath $ErrorFileName + gcloud storage cp $ErrorFileName "gs://${bucket}/${bucket_folder}/${machine_type}/" + exit 1 + } + catch + { + # Expected. Successfully removed from path. + gcloud storage cp $SuccessFileName "gs://${bucket}/${bucket_folder}/${machine_type}/" + } + } + catch + { + Write-Output $_ + Write-Output $_.ScriptStackTrace + + "Error. See VM serial port 1 logs for details." | Out-File -FilePath $ErrorFileName + gcloud storage cp $ErrorFileName "gs://${bucket}/${bucket_folder}/${machine_type}/" + } +} + +Perform-Test -JdkVersion 8 +Perform-Test -JdkVersion 11 +Perform-Test -JdkVersion 17 +Perform-Test -JdkVersion 19 +Perform-Test -JdkVersion 20 + +# Store the VM's console logs +gcloud compute instances get-serial-port-output "${vm_name}" --zone "${vm_zone}" > "${vm_name}.txt" +gcloud storage cp "${vm_name}.txt" "gs://${bucket}/${bucket_folder}/logs/${vm_name}.txt" +"Done with JDK testing." diff --git a/temurin-install-testing/startup.sh b/temurin-install-testing/startup.sh new file mode 100644 index 0000000000..2c833fcd1f --- /dev/null +++ b/temurin-install-testing/startup.sh @@ -0,0 +1,144 @@ +#!/bin/bash + +# +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +function prepare_installer_debian_ubuntu { + # [START debian_adoptium_key] + sudo mkdir -p /etc/apt/keyrings + sudo wget -O - https://packages.adoptium.net/artifactory/api/gpg/key/public | + sudo tee /etc/apt/keyrings/adoptium.asc + # [END debian_adoptium_key] + +# [START debian_adoptium_repo] +eval "$(grep VERSION_CODENAME /etc/os-release)" +sudo tee /etc/apt/sources.list.d/adoptium.list </dev/null; then + echo "Java already installed." + exit 1 + fi + + $INSTALLER update -y + if ! $INSTALLER install -y "$1" 2>"result.txt"; then + FILE=$ERROR_FILE + else + # java -version uses the error stream + if ! java -version 2>"result.txt"; then + FILE=$ERROR_FILE + fi + fi + cat "result.txt" >"$FILE" + gcloud storage cp "$FILE" "gs://${bucket}/${bucket_folder}/${machine_type}/" + $INSTALLER remove -y "$1" +} + +prepare_installer +perform_test temurin-8-jdk +perform_test temurin-11-jdk +perform_test temurin-17-jdk +perform_test temurin-19-jdk +perform_test temurin-20-jdk + +# Store the VM's console logs +gcloud compute instances get-serial-port-output "${vm_name}" --zone "${vm_zone}" >"${vm_name}.txt" +gcloud storage cp "${vm_name}.txt" "gs://${bucket}/${bucket_folder}/logs/${vm_name}.txt" diff --git a/temurin-install-testing/test-all.sh b/temurin-install-testing/test-all.sh new file mode 100755 index 0000000000..3e080fd90c --- /dev/null +++ b/temurin-install-testing/test-all.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -eo pipefail + +function wait { + echo "$(date +%T) Sleeping 20 min" + sleep 1200 +} + +terraform apply -var="enable_arm=true" -auto-approve +wait +BUCKET_FOLDER=$(terraform output --raw bucket_folder) +terraform apply -auto-approve +terraform apply -var="enable_linux=true" -var="bucket_folder=$BUCKET_FOLDER" -auto-approve +# For single invocations: terraform apply -var="enable_linux=true" -auto-approve +wait +terraform apply -auto-approve +terraform apply -var="enable_windows=true" -var="bucket_folder=$BUCKET_FOLDER" -auto-approve +# For single invocations: terraform apply -var="enable_windows=true" -auto-approve +wait +terraform apply -auto-approve diff --git a/temurin-install-testing/variables.tf b/temurin-install-testing/variables.tf new file mode 100644 index 0000000000..7a5be9055b --- /dev/null +++ b/temurin-install-testing/variables.tf @@ -0,0 +1,153 @@ +variable "user_email" { + type = string + description = "Current user email" +} +variable "project_id" { + type = string + description = "GCP Project ID of the project being used" +} +variable "bucket" { + type = string + description = "Result storage bucket" +} + +variable "enable_arm" { + type = bool + description = "If true, perform arm tests" + default = false +} +variable "enable_windows" { + type = bool + description = "If true, perform x86 Windows tests" + default = false +} +variable "enable_linux" { + type = bool + description = "If true, perform x86 Linux tests" + default = false +} + +variable "location" { + type = string + description = "Bucket location used for GCS" + default = "US-CENTRAL1" +} +variable "region" { + type = string + description = "GCP region used to deploy resources" + default = "us-central1" +} +variable "zone" { + type = string + description = "GCP zone used to deploy resources. Must be a zone in the chosen region." + default = "us-central1-a" +} +variable "bucket_folder" { + type = string + description = "Generally, leave empty. By default, a timestamp will be provided as the value." + default = "" +} + + +## See https://cloud.google.com/compute/docs/machine-resource +variable "x86_machine_types" { + type = list(string) + description = "GCE machine types to create instances of" + default = [ + #"c3-highcpu-4", + #"n2-standard-96", + ##"m3-ultramem-32", # Requires special quota + #"n2-standard-2", + #"c2-standard-4", + ##"m2-ultramem-208", # Requires special quota + #"a2-highgpu-1g", + ##"m1-megamem-96", # Requires special quota + ##"m1-ultramem-40", # Requires special quota + "n1-standard-1", + #"t2d-standard-1", + #"n2d-standard-2", + ] +} + +# See https://cloud.google.com/compute/docs/images +variable "x86_boot_images" { + type = list(string) + description = "GCE boot image to use with created instance" + default = [ + "debian-cloud/debian-11", + "debian-cloud/debian-10", + + "rhel-cloud/rhel-9", + "rhel-cloud/rhel-7", + "rhel-sap-cloud/rhel-9-0-sap-ha", + "rhel-sap-cloud/rhel-7-7-sap-ha", + + "centos-cloud/centos-stream-9", + "centos-cloud/centos-stream-8", + "centos-cloud/centos-7", + + "rocky-linux-cloud/rocky-linux-9-optimized-gcp", + "rocky-linux-cloud/rocky-linux-8-optimized-gcp", + "rocky-linux-cloud/rocky-linux-8", + + "ubuntu-os-cloud/ubuntu-2204-lts", + "ubuntu-os-cloud/ubuntu-2004-lts", + "ubuntu-os-pro-cloud/ubuntu-pro-2204-lts", + "ubuntu-os-pro-cloud/ubuntu-pro-1804-lts", + "ubuntu-os-pro-cloud/ubuntu-pro-1604-lts", + + "suse-cloud/sles-15", + "suse-byos-cloud/sles-12-byos", + "suse-sap-cloud/sles-12-sp5-sap", + ] +} + + +# See https://cloud.google.com/compute/docs/images +variable "x86_windows_boot_images" { + type = list(string) + description = "GCE Windows boot image to use with created instance" + default = [ + "windows-cloud/windows-2022", + "windows-cloud/windows-2022-core", + "windows-cloud/windows-2012-r2", + "windows-cloud/windows-2012-r2-core", + + "windows-sql-cloud/sql-web-2022-win-2022", + "windows-sql-cloud/sql-std-2022-win-2022", + "windows-sql-cloud/sql-ent-2022-win-2022", + + "windows-sql-cloud/sql-web-2022-win-2019", + "windows-sql-cloud/sql-std-2022-win-2019", + "windows-sql-cloud/sql-ent-2022-win-2019", + + "windows-sql-cloud/sql-web-2014-win-2012-r2", + "windows-sql-cloud/sql-std-2014-win-2012-r2", + "windows-sql-cloud/sql-ent-2014-win-2012-r2", + ] +} + +## See https://cloud.google.com/compute/docs/machine-resource +variable "arm_machine_types" { + type = list(string) + description = "GCE machine types to create instances of" + default = [ + "t2a-standard-1" + ] +} + +# See https://cloud.google.com/compute/docs/images +variable "arm_boot_images" { + type = list(string) + description = "GCE boot image to use with created instance" + default = [ + "debian-cloud/debian-11-arm64", + "rhel-cloud/rhel-9-arm64", + "rocky-linux-cloud/rocky-linux-9-arm64", + "rocky-linux-cloud/rocky-linux-9-optimized-gcp-arm64", + "rocky-linux-cloud/rocky-linux-8-optimized-gcp-arm64", + "suse-cloud/sles-15-arm64", + "ubuntu-os-cloud/ubuntu-2204-lts-arm64", + "ubuntu-os-cloud/ubuntu-2004-lts-arm64", + ] +}