NationalSecurityAgency · ivakegg · Nov 25, 2024 · Nov 25, 2024 · Nov 25, 2024 · Nov 25, 2024
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -74,7 +74,7 @@ jobs:
         USER_NAME: ${{ secrets.USER_NAME }}
         ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }}
       run: |
-        RUN_TESTS="mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -V -B -e -ntp "-Dstyle.color=always" -Pdev,examples,assemble,spotbugs -Dmaven.build.cache.enabled=false -Ddeploy -Ddist -T1C clean verify"
+        RUN_TESTS="mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -V -B -e -ntp "-Dstyle.color=always" -Pdev,examples,assemble,spotbugs -Dmaven.build.cache.enabled=false -Ddeploy -Ddist -T1C clean verify -Dutils -Dservices -Dstarters"
         $RUN_TESTS \
           || { echo "***** TESTS FAILED. Attempting retry."; $RUN_TESTS; } \
           || { echo "***** TESTS FAILED. Attempting final retry."; $RUN_TESTS; }
@@ -110,14 +110,14 @@ jobs:
   #         USER_NAME: ${{ secrets.USER_NAME }}
   #         ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }}
   #       run: |
-  #         BUILD="mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -V -B -e -Pdev,examples,assemble,spotbugs -Dmaven.build.cache.enabled=false -DskipServices -Ddeploy -Ddist -T1C -pl "-:config-service" clean install -DskipTests"
+  #         BUILD="mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -V -B -e -Dstarters -Dservices -Dutils -Pdev,examples,assemble,spotbugs -Dmaven.build.cache.enabled=false -DonlyServiceApis -Ddeploy -Ddist -T1C -pl "-:config-service" clean install -DskipTests"
   #         $BUILD
   #     - name: Run Microservice Unit Tests
   #       env:
   #         USER_NAME: ${{ secrets.USER_NAME }}
   #         ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }}
   #       run: |
-  #         RUN_TESTS="mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -V -B -e -Dmaven.build.cache.enabled=false verify"
+  #         RUN_TESTS="mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -V -B -e -Dstarters -Dservices -Dutils -Dmaven.build.cache.enabled=false verify"
   #         cd microservices
   #         $RUN_TESTS \
   #           || { echo "***** TESTS FAILED. Attempting retry."; $RUN_TESTS; } \
@@ -183,13 +183,13 @@ jobs:
           ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }}
         run: |
           # update datawave dependencies to use the latest snapshots
-          mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -B -V -e versions:update-properties versions:update-parent -DallowSnapshots=true -Dincludes=gov.nsa.*
+          mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -B -V -e -Dstarters -Dservices -Dutils versions:update-properties versions:update-parent -DallowSnapshots=true -Dincludes=gov.nsa.*
 
-          mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -B -V -e -Pcompose -Dmicroservice-docker -Dquickstart-docker -Ddeploy -Dtar -DskipTests clean install
+          mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -B -V -e -Dstarters -Dservices -Dutils -Pcompose -Dmicroservice-docker -Dquickstart-docker -Ddeploy -Dtar -DskipTests clean install
 
           # free up some space so that we don't run out
           docker system prune -f
-          mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -B -V -e -Pcompose -Dmicroservice-docker -Dquickstart-docker -Ddeploy -Dtar -DskipTests clean
+          mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -B -V -e -Dstarters -Dservices -Dutils -Pcompose -Dmicroservice-docker -Dquickstart-docker -Ddeploy -Dtar -DskipTests clean
 
           cd docker
           ./bootstrap.sh
@@ -264,10 +264,10 @@ jobs:
                      -Durl.hadoop=https://bogus.apache.org/hadoop/common/hadoop-3.3.6/hadoop-3.3.6.tar.gz \
                      -Durl.maven=https://bogus.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz"
 
-          mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -B -V -e -Pcompose -Dmicroservice-docker -Dquickstart-docker -Dquickstart-maven ${DIST_URLS} -Ddeploy -Dtar -DskipTests -Dmaven.build.cache.enabled=false clean install
+          mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -B -V -e -Dstarters -Dservices -Dutils -Pcompose -Dmicroservice-docker -Dquickstart-docker -Dquickstart-maven ${DIST_URLS} -Ddeploy -Dtar -DskipTests -Dmaven.build.cache.enabled=false clean install
           # free up some space so that we don't run out
           docker system prune -f
-          mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -B -V -e -Pcompose -Dmicroservice-docker -Dquickstart-docker -Dquickstart-maven ${DIST_URLS} -Ddeploy -Dtar -DskipTests -Dmaven.build.cache.enabled=false clean
+          mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -B -V -e -Dstarters -Dservices -Dutils -Pcompose -Dmicroservice-docker -Dquickstart-docker -Dquickstart-maven ${DIST_URLS} -Ddeploy -Dtar -DskipTests -Dmaven.build.cache.enabled=false clean
           cd docker
           ./bootstrap.sh
           attempt=0

diff --git a/.gitmodules b/.gitmodules
@@ -1,90 +0,0 @@
-[submodule "microservices/services/authorization"]
-	path = microservices/services/authorization
-	url = [email protected]:NationalSecurityAgency/datawave-authorization-service.git
-[submodule "core/in-memory-accumulo"]
-	path = core/in-memory-accumulo
-	url = [email protected]:NationalSecurityAgency/datawave-in-memory-accumulo.git
-[submodule "core/utils/accumulo-utils"]
-	path = core/utils/accumulo-utils
-	url = [email protected]:NationalSecurityAgency/datawave-accumulo-utils.git
-[submodule "core/utils/common-utils"]
-	path = core/utils/common-utils
-	url = [email protected]:NationalSecurityAgency/datawave-common-utils.git
-[submodule "core/base-rest-responses"]
-	path = core/base-rest-responses
-	url = [email protected]:NationalSecurityAgency/datawave-base-rest-responses.git
-[submodule "core/utils/type-utils"]
-	path = core/utils/type-utils
-	url = [email protected]:NationalSecurityAgency/datawave-type-utils.git
-[submodule "core/utils/metadata-utils"]
-	path = core/utils/metadata-utils
-	url = [email protected]:NationalSecurityAgency/datawave-metadata-utils.git
-[submodule "microservices/services/accumulo"]
-	path = microservices/services/accumulo
-	url = [email protected]:NationalSecurityAgency/datawave-accumulo-service.git
-[submodule "microservices/services/dictionary"]
-	path = microservices/services/dictionary
-	url = [email protected]:NationalSecurityAgency/datawave-dictionary-service.git
-[submodule "microservices/services/audit"]
-	path = microservices/services/audit
-	url = [email protected]:NationalSecurityAgency/datawave-audit-service.git
-[submodule "microservices/services/query-metric"]
-	path = microservices/services/query-metric
-	url = [email protected]:NationalSecurityAgency/datawave-query-metric-service.git
-[submodule "core/metrics-reporter"]
-	path = core/metrics-reporter
-	url = [email protected]:NationalSecurityAgency/datawave-metrics-reporter.git
-[submodule "microservices/starters/cache"]
-	path = microservices/starters/cache
-	url = [email protected]:NationalSecurityAgency/datawave-spring-boot-starter-cache.git
-[submodule "microservices/starters/datawave"]
-	path = microservices/starters/datawave
-	url = [email protected]:NationalSecurityAgency/datawave-spring-boot-starter.git
-[submodule "microservices/services/hazelcast"]
-	path = microservices/services/hazelcast
-	url = [email protected]:NationalSecurityAgency/datawave-hazelcast-service.git
-[submodule "microservices/starters/metadata"]
-	path = microservices/starters/metadata
-	url = [email protected]:NationalSecurityAgency/datawave-spring-boot-starter-metadata.git
-[submodule "microservices/starters/query-metric"]
-	path = microservices/starters/query-metric
-	url = [email protected]:NationalSecurityAgency/datawave-spring-boot-starter-query-metric.git
-[submodule "microservices/starters/audit"]
-	path = microservices/starters/audit
-	url = [email protected]:NationalSecurityAgency/datawave-spring-boot-starter-audit.git
-[submodule "microservices/services/config"]
-	path = microservices/services/config
-	url = [email protected]:NationalSecurityAgency/datawave-config-service.git
-[submodule "microservices/microservice-parent"]
-	path = microservices/microservice-parent
-	url = [email protected]:NationalSecurityAgency/datawave-parent.git
-[submodule "microservices/microservice-service-parent"]
-	path = microservices/microservice-service-parent
-	url = [email protected]:NationalSecurityAgency/datawave-service-parent.git
-[submodule "contrib/datawave-utils"]
-	path = contrib/datawave-utils
-	url = [email protected]:NationalSecurityAgency/datawave-utils.git
-[submodule "microservices/starters/query"]
-	path = microservices/starters/query
-	url = [email protected]:NationalSecurityAgency/datawave-spring-boot-starter-query.git
-[submodule "microservices/services/query"]
-	path = microservices/services/query
-	url = [email protected]:NationalSecurityAgency/datawave-query-service.git
-[submodule "microservices/services/query-executor"]
-	path = microservices/services/query-executor
-	url = [email protected]:NationalSecurityAgency/datawave-query-executor-service.git
-[submodule "microservices/services/modification"]
-	path = microservices/services/modification
-	url = [email protected]:NationalSecurityAgency/datawave-modification-service.git
-[submodule "microservices/services/mapreduce-query"]
-	path = microservices/services/mapreduce-query
-	url = [email protected]:NationalSecurityAgency/datawave-mapreduce-query-service.git
-[submodule "microservices/starters/cached-results"]
-	path = microservices/starters/cached-results
-	url = [email protected]:NationalSecurityAgency/datawave-spring-boot-starter-cached-results.git
-[submodule "microservices/services/map"]
-	path = microservices/services/map
-	url = [email protected]:NationalSecurityAgency/datawave-map-service.git
-[submodule "microservices/services/file-provider"]
-	path = microservices/services/file-provider
-	url = [email protected]:NationalSecurityAgency/datawave-file-provider-service.git

diff --git a/BUILDME.md b/BUILDME.md
@@ -65,7 +65,7 @@ mvn -Pdev,assemble,rpm -Ddeploy -Dtar -Ddist -DskipTests clean install
 Datawave web services utilize several microservices at runtime (currently authorization and auditing, although that
 list will expand soon). Datawave depends on api modules for some of these services, and the dependencies are set in
 the parent pom (see `version.datawave.*` properties) to released versions. If you wish to build the microservices
-for some reason, you can simply add `-Dservices` to your maven build command.
+for some reason, you can simply add `-Dservices` to your maven build command.  If you wish to build the starters you can add `-Dstarters` and for the utility modules add `-Dutils`.
 
 ### Releasing Microservices
 

diff --git a/README.md b/README.md
@@ -19,42 +19,74 @@ Basic build instructions are [here](BUILDME.md)
 
 ## How to Use this Repository
 
-The microservices and associated utility projects are intended to be
-developed, versioned, and released independently and as such are stored
-in separate repositories. This repository includes them all as submodules
-in order to provide an easy way to import them all in an IDE for viewing
-the code, or refactoring. [Git submodules](https://git-scm.com/book/en/v2/Git-Tools-Submodules)
-require some extra commands over the normal ones that one may be familiar
-with.
-
-### Cloning with all submodules
-Cloning with all of the submodules is not required; however, if you are interested in checking 
-out and building all of the datawave projects under one repo, read this!
-
-It's easiest to clone the repository pointing the submodules at the same branch
-```bash
-# Start out by cloning the project as you normally would.
-git clone [email protected]:NationalSecurityAgency/datawave.git
-
-# Now, use git to retrieve all of the datawave submodules.
-# This will leave your submodules in a detached head state.
-cd datawave
-git submodule update --init --recursive
-
-# You can checkout the main branch for each submodule so that you are no longer in a detached head state.
-# The addition of `|| :` will ensure that the command is executed for each submodule, 
-# ignoring failures for submodules that don't have a main branch.
-git submodule foreach 'git checkout main || :'
-
-# It is recommended to build the project using multiple threads.
-mvn -Pdocker,dist clean install -T 1C
+The microservices and associated utility projects are intended to be developed, versioned,
+and released independently.  The following subdirectories contain those independently
+versioned modules:
+
+```
+core/utils/type-utils
+contrib/datawave-utils
+core/base-rest-responses
+core/in-memory-accumulo
+core/metrics-reporter
+core/utils/accumulo-utils
+core/utils/common-utils
+core/utils/metadata-utils
+microservices/microservice-parent
+microservices/microservice-service-parent
+microservices/starters/audit
+microservices/starters/cache
+microservices/starters/cached-results
+microservices/starters/datawave
+microservices/starters/metadata
+microservices/starters/query
+microservices/starters/query-metric
+microservices/services/accumulo
+microservices/services/audit
+microservices/services/authorization
+microservices/services/config
+microservices/services/dictionary
+microservices/services/file-provider
+microservices/services/hazelcast
+microservices/services/map
+microservices/services/mapreduce-query
+microservices/services/modification
+microservices/services/query
+microservices/services/query-executor
+microservices/services/query-metric
+```
 
-# If you don't want to build the microservices, you can skip them.
-mvn -Pdocker,dist -DskipMicroservices clean install -T 1C
+Each of those subdirectories contain a .gitrepo file that keeps track of where the code came from.
 
-# If you decide that you no longer need the submodules, you can remove them.
-git submodule deinit --all
+### Updating one of the datawave sub-repositories
+At one point we used submodules to link in a all of the sub-repositories.  We have now switched
+to including the submodules' code directly into the main datawave repository.  The git subrepo
+mechanism (https://github.com/ingydotnet/git-subrepo) was used to facilitate the transition.
+That same mechanism can be used to pull in changes from the other repositories as needed until
+they can be removed altogether.  The original cloning of the sub repositories was done using
+the subrepo command as follows:
 ```
+git subrepo clone <repo> <dir>
+```
+If changes need to be pulled in, then the following process can be used:
+```
+git subrepo pull <dir>
+```
+### Building
+
+It is recommended to build the project using multiple threads.  This will not build the starters, utilities, and services.
+```
+mvn -Pdocker,dist clean install -T 1C
+```
+
+If you want to build the starters, util modules, and services as well then try this
+```
+mvn -Pdocker,dist -Dstarters -Dservices -Dutils clean install -T 1C
+```
+If you want to build the service apis but not the services themselveds then add -DonlyServiceApis
+
+NOTE: The util modules, starters, and services are actually tagged and deployed separately.
+  Hence the snapshot versions within those sub repos are not connected together.
 
 ### DataWave Microservices
 

diff --git a/contrib/datawave-utils b/contrib/datawave-utils
diff --git a/contrib/datawave-utils/.gitignore b/contrib/datawave-utils/.gitignore
@@ -0,0 +1,9 @@
+target/
+
+.idea/
+*.iml
+*.iws
+
+.classpath
+.project
+.settings/
diff --git a/contrib/datawave-utils/.gitrepo b/contrib/datawave-utils/.gitrepo
@@ -0,0 +1,12 @@
+; DO NOT EDIT (unless you know what you are doing)
+;
+; This subdirectory is a git "subrepo", and this file is maintained by the
+; git-subrepo command. See https://github.com/ingydotnet/git-subrepo#readme
+;
+[subrepo]
+	remote = [email protected]:NationalSecurityAgency/datawave-utils.git
+	branch = main
+	commit = 4348fc36a3519ca9e5d1b96ac47c3f0b64abe34b
+	parent = 1f96d1af84c45f32007da0105b8eab514ec3f7d3
+	method = merge
+	cmdver = 0.4.9