Added cosv database (#2884)

saveourtool · Dec 20, 2023 · 58898ec · 58898ec
1 parent 7db51e3
commit 58898ec
Show file tree

Hide file tree

Showing 22 changed files with 426 additions and 1 deletion.
diff --git a/buildSrc/src/main/kotlin/com/saveourtool/save/buildutils/DockerStackConfiguration.kt b/buildSrc/src/main/kotlin/com/saveourtool/save/buildutils/DockerStackConfiguration.kt
@@ -40,11 +40,17 @@ fun Project.registerLiquibaseTask(profile: String) {
         relativeChangeLogFile = "save-demo/db/db.changelog-demo.xml",
         profile = profile
     )
+    val registerLiquibaseTaskCosv = registerLiquibaseTask(
+        projectName = "save-cosv",
+        relativeChangeLogFile = "save-cosv/db/db.changelog-cosv.xml",
+        profile = profile
+    )
     tasks.register("liquibaseUpdate") {
         dependsOn(
             registerLiquibaseTaskBackend,
             registerLiquibaseTaskSandbox,
-            registerLiquibaseTaskDemo
+            registerLiquibaseTaskDemo,
+            registerLiquibaseTaskCosv,
         )
     }
 }

diff --git a/save-cloud-charts/save-cloud/values.yaml b/save-cloud-charts/save-cloud/values.yaml
@@ -78,6 +78,7 @@ mysql:
   backend_schema: save_cloud
   sandbox_schema: save_sandbox
   demo_schema: save_demo
+  cosv_schema: cosv
   root_password: '123'
   migrations:
     # Whether database migrations should be executed while deploying the application

diff --git a/save-cosv/db/db.changelog-cosv.xml b/save-cosv/db/db.changelog-cosv.xml
@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<databaseChangeLog
+        xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog
+                      http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.8.xsd">
+
+    <preConditions>
+        <dbms type="mysql"/>
+    </preConditions>
+
+    <changeSet id="cosv-init-changelog" author="frolov">
+        <tagDatabase tag="db_cosv_init"/>
+    </changeSet>
+
+    <include file="tables/_all_changelogs.xml" relativeToChangelogFile="true"/>
+    <include file="test-data/db.changelog-insert.xml" relativeToChangelogFile="true" context="dev"/>
+
+</databaseChangeLog>
diff --git a/save-cosv/db/tables/_all_changelogs.xml b/save-cosv/db/tables/_all_changelogs.xml
@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<databaseChangeLog
+        xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog
+                      http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.8.xsd">
+
+    <include file="raw-cosv-file.xml" relativeToChangelogFile="true"/>
+    <include file="cosv-file.xml" relativeToChangelogFile="true"/>
+    <include file="vulnerability-metadata.xml" relativeToChangelogFile="true"/>
+    <include file="lnk-vulnerability-metadata-tag.xml" relativeToChangelogFile="true"/>
+    <include file="lnk-vulnerability-metadata-user.xml" relativeToChangelogFile="true"/>
+    <include file="vulnerability-metadata-project.xml" relativeToChangelogFile="true"/>
+    <include file="cosv-generated-id.xml" relativeToChangelogFile="true"/>
+
+    <changeSet id="cosv-tag" author="frolov">
+        <tagDatabase tag="cosv-tables"/>
+    </changeSet>
+
+</databaseChangeLog>
diff --git a/save-cosv/db/tables/cosv-file.xml b/save-cosv/db/tables/cosv-file.xml
@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<databaseChangeLog
+        xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog
+                      http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.8.xsd">
+
+    <changeSet id="cosv-file-creation" author="saveourtool-dev">
+        <createTable tableName="cosv_file">
+            <column name="id" type="bigint" autoIncrement="true">
+                <constraints primaryKey="true" nullable="false"/>
+            </column>
+            <column name="identifier" type="varchar(250)">
+                <constraints nullable="false"/>
+            </column>
+            <column name="modified" type="DATETIME(3)">
+                <constraints nullable="false"/>
+            </column>
+            <column name="prev_cosv_file_id" type="bigint">
+                <constraints foreignKeyName="fk_cosv_file_prev_cosv_file" references="cosv_file(id)"
+                             nullable="true" deleteCascade="true"/>
+            </column>
+        </createTable>
+        <addUniqueConstraint tableName="cosv_file" columnNames="identifier,modified" constraintName="uq_osv_file"/>
+    </changeSet>
+
+</databaseChangeLog>
diff --git a/save-cosv/db/tables/cosv-generated-id.xml b/save-cosv/db/tables/cosv-generated-id.xml
@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<databaseChangeLog
+        xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog
+                      http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.8.xsd">
+
+    <changeSet id="cosv-generated-id-creation" author="saveourtool-dev">
+        <createTable tableName="cosv_generated_id">
+            <column name="id" type="bigint" autoIncrement="true">
+                <constraints primaryKey="true" nullable="false"/>
+            </column>
+            <column name="create_date" type="DATETIME(3)">
+                <constraints nullable="false"/>
+            </column>
+            <column name="update_date" type="DATETIME(3)">
+                <constraints nullable="false"/>
+            </column>
+        </createTable>
+    </changeSet>
+
+</databaseChangeLog>
diff --git a/save-cosv/db/tables/lnk-vulnerability-metadata-tag.xml b/save-cosv/db/tables/lnk-vulnerability-metadata-tag.xml
@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<databaseChangeLog
+        xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog
+                      http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.8.xsd">
+
+    <changeSet id="lnk-vulnerability-metadata-tag-creation" author="saveourtool-dev">
+        <createTable tableName="lnk_vulnerability_metadata_tag">
+            <column name="id" type="bigint" autoIncrement="true">
+                <constraints primaryKey="true" nullable="false"/>
+            </column>
+            <column name="vulnerability_metadata_id" type="bigint">
+                <constraints foreignKeyName="fk_lnk_vulnerability_metadata_tag_vulnerability_metadata" references="vulnerability_metadata(id)"
+                             nullable="false" deleteCascade="true"/>
+            </column>
+            <column name="tag_id" type="bigint">
+                <constraints foreignKeyName="fk_lnk_vulnerability_metadata_tag_tag" references="save_cloud.tag(id)"
+                             nullable="false" deleteCascade="true"/>
+            </column>
+        </createTable>
+
+        <addUniqueConstraint tableName="lnk_vulnerability_metadata_tag" columnNames="tag_id,vulnerability_metadata_id"
+                             constraintName="uq_lnk_vulnerability_metadata_tag"/>
+    </changeSet>
+
+</databaseChangeLog>
diff --git a/save-cosv/db/tables/lnk-vulnerability-metadata-user.xml b/save-cosv/db/tables/lnk-vulnerability-metadata-user.xml
@@ -0,0 +1,26 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<databaseChangeLog
+        xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog
+                      http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.8.xsd">
+
+    <changeSet id="lnk-vulnerability-metadata-user-creation" author="saveourtool-dev">
+        <createTable tableName="lnk_vulnerability_metadata_user">
+            <column name="id" type="bigint" autoIncrement="true">
+                <constraints primaryKey="true" nullable="false"/>
+            </column>
+            <column name="vulnerability_metadata_id" type="bigint">
+                <constraints foreignKeyName="fk_vulnerability_metadata_user_vulnerability_metadata" references="vulnerability_metadata(id)"
+                             nullable="false" deleteCascade="true"/>
+            </column>
+            <column name="user_id" type="bigint">
+                <constraints foreignKeyName="fk_lnk_vulnerability_metadata_user_user" references="save_cloud.user(id)"
+                             nullable="false" deleteCascade="true"/>
+            </column>
+        </createTable>
+
+        <addUniqueConstraint tableName="lnk_vulnerability_metadata_user" columnNames="user_id,vulnerability_metadata_id"/>
+    </changeSet>
+
+</databaseChangeLog>
diff --git a/save-cosv/db/tables/raw-cosv-file.xml b/save-cosv/db/tables/raw-cosv-file.xml
@@ -0,0 +1,52 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<databaseChangeLog
+        xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog
+                      http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.8.xsd">
+
+    <changeSet id="raw-cosv-file-creation" author="saveourtool-dev">
+        <createTable tableName="raw_cosv_file">
+            <column name="id" type="bigint" autoIncrement="true">
+                <constraints primaryKey="true" nullable="false"/>
+            </column>
+            <column name="file_name" type="varchar(250)">
+                <constraints nullable="false"/>
+            </column>
+            <column name="user_id" type="bigint">
+                <constraints foreignKeyName="fk_raw_cosv_file_user" references="save_cloud.user(id)" nullable="false"/>
+            </column>
+            <column name="organization_id" type="bigint">
+                <constraints foreignKeyName="fk_raw_cosv_organization" references="save_cloud.organization(id)" nullable="false"/>
+            </column>
+            <column name="status" type="varchar(64)">
+                <constraints nullable="false"/>
+            </column>
+            <column name="status_message" type="TEXT">
+                <constraints nullable="true"/>
+            </column>
+            <column name="content_length" type="bigint">
+                <constraints nullable="true"/>
+            </column>
+            <column name="create_date" type="DATETIME(3)">
+                <constraints nullable="false"/>
+            </column>
+            <column name="update_date" type="DATETIME(3)">
+                <constraints nullable="false"/>
+            </column>
+        </createTable>
+        <addUniqueConstraint tableName="raw_cosv_file" columnNames="file_name,organization_id,user_id"
+                             constraintName="uq_raw_osv_file_file_name_per_organization_and_user"/>
+    </changeSet>
+
+    <changeSet id="raw-cosv-file-events" author="saveourtool-dev" context="prod">
+        <sql>
+            CREATE EVENT delete_processed_raw_cosv_file
+            ON SCHEDULE EVERY 1 DAY
+            STARTS TIMESTAMP(CURRENT_DATE + INTERVAL 1 HOUR)
+            DO DELETE FROM raw_cosv_file WHERE DATEDIFF(NOW(), update_date) >= 2;
+        </sql>
+    </changeSet>
+
+
+</databaseChangeLog>
diff --git a/save-cosv/db/tables/vulnerability-metadata-project.xml b/save-cosv/db/tables/vulnerability-metadata-project.xml
@@ -0,0 +1,26 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<databaseChangeLog
+        xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog
+                      http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.8.xsd">
+
+    <changeSet id="vulnerability-metadata-project-creation" author="saveourtool-dev">
+        <createTable tableName="vulnerability_metadata_project">
+            <column name="id" type="bigint" autoIncrement="true">
+                <constraints primaryKey="true" nullable="false"/>
+            </column>
+            <column name="name" type="varchar(250)">
+                <constraints nullable="false"/>
+            </column>
+            <column name="url" type="varchar(250)"/>
+            <column name="vulnerability_metadata_id" type="bigint">
+                <constraints foreignKeyName="fk_vulnerability_metadata_project_vulnerabilitymetadata" references="vulnerability_metadata(id)"
+                             nullable="false" deleteCascade="true"/>
+            </column>
+            <column name="versions" type="varchar(250)" defaultValue="null"/>
+            <column name="type" type="varchar(64)" defaultValue="PROJECT"/>
+        </createTable>
+    </changeSet>
+
+</databaseChangeLog>
diff --git a/save-cosv/db/tables/vulnerability-metadata.xml b/save-cosv/db/tables/vulnerability-metadata.xml
@@ -0,0 +1,54 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<databaseChangeLog
+        xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog
+                      http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.8.xsd">
+
+    <changeSet id="vulnerability-metadata-creation" author="saveourtool-dev">
+        <createTable tableName="vulnerability_metadata">
+            <column name="id" type="bigint" autoIncrement="true">
+                <constraints primaryKey="true" nullable="false"/>
+            </column>
+            <column name="identifier" type="varchar(100)">
+                <constraints nullable="false"/>
+            </column>
+            <column name="summary" type="varchar(250)">
+                <constraints nullable="false"/>
+            </column>
+            <column name="details" type="TEXT">
+                <constraints nullable="false"/>
+            </column>
+            <column name="severity_num" type="DECIMAL(3,1)">
+                <constraints nullable="false"/>
+            </column>
+            <column name="submitted" type="DATETIME(3)">
+                <constraints nullable="true"/>
+            </column>
+            <column name="modified" type="DATETIME(3)">
+                <constraints nullable="true"/>
+            </column>
+            <column name="user_id" type="bigint">
+                <constraints foreignKeyName="fk_cosv_metadata_user" references="save_cloud.user(id)"
+                             nullable="false" deleteCascade="true"/>
+            </column>
+            <column name="organization_id" type="bigint">
+                <constraints foreignKeyName="fk_cosv_metadata_organization" references="save_cloud.organization(id)"
+                             nullable="true" deleteCascade="true"/>
+            </column>
+            <column name="language" type="varchar(64)">
+                <constraints nullable="false"/>
+            </column>
+            <column name="status" type="varchar(64)" defaultValue="CREATED">
+                <constraints nullable="false"/>
+            </column>
+            <column name="latest_cosv_file_id" type="bigint">
+                <constraints foreignKeyName="fk_vulnerability_metadata_latest_cosv_file" references="cosv_file(id)"
+                             nullable="false" deleteCascade="true"/>
+            </column>
+        </createTable>
+        <addUniqueConstraint tableName="vulnerability_metadata" columnNames="identifier"
+                             constraintName="uq_vulnerability_metadata_identifier"/>
+    </changeSet>
+
+</databaseChangeLog>
diff --git a/save-cosv/db/test-data/cosv-file-insert.xml b/save-cosv/db/test-data/cosv-file-insert.xml
@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<databaseChangeLog
+        xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog
+                      http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.8.xsd">
+
+    <changeSet id="cosv-file-insert" author="nulls" context="dev">
+        <loadData tableName="cosv_file" separator=";" file="save-cosv/db/test-data/sqlRequests/cosv-file.csv">
+            <column header="id" name="id" type="NUMERIC" />
+            <column header="identifier" name="identifier" type="varchar(250)" />
+            <column header="modified" name="modified" type="DATE" />
+            <column header="prev_cosv_file_id" name="prev_cosv_file_id" type="NUMERIC"/>
+        </loadData>
+    </changeSet>
+
+</databaseChangeLog>
diff --git a/save-cosv/db/test-data/db.changelog-insert.xml b/save-cosv/db/test-data/db.changelog-insert.xml
@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<databaseChangeLog
+        xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog
+                      http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.8.xsd">
+
+    <include file="cosv-file-insert.xml" relativeToChangelogFile="true"/>
+    <include file="vulnerability-metadata-insert.xml" relativeToChangelogFile="true"/>
+    <include file="vulnerability-metadata-project-insert.xml" relativeToChangelogFile="true"/>
+    <include file="lnk-vulnerability-metadata-user-insert.xml" relativeToChangelogFile="true"/>
+
+    <changeSet id="add-cosv-tests-data" author="frolov">
+        <tagDatabase tag="cosv-test-data-insert"/>
+    </changeSet>
+
+</databaseChangeLog>
diff --git a/save-cosv/db/test-data/lnk-vulnerability-metadata-user-insert.xml b/save-cosv/db/test-data/lnk-vulnerability-metadata-user-insert.xml
@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<databaseChangeLog
+        xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog
+                      http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.8.xsd">
+
+    <changeSet id="lnk-vulnerability-metadata-user-insert-1" author="nulls" context="dev">
+
+        <loadData tableName="lnk_vulnerability_metadata_user" encoding="UTF-8" separator=";" quotchar="&quot;" file="save-cosv/db/test-data/sqlRequests/lnk-vulnerability-metadata-user.csv">
+            <column header="id" name="id" type="NUMERIC" />
+            <column header="vulnerability_metadata_id" name="vulnerability_metadata_id" type="NUMERIC" />
+            <column header="user_id" name="user_id" type="NUMERIC" />
+        </loadData>
+
+    </changeSet>
+
+</databaseChangeLog>
diff --git a/save-cosv/db/test-data/sqlRequests/cosv-file.csv b/save-cosv/db/test-data/sqlRequests/cosv-file.csv
@@ -0,0 +1,6 @@
+id;identifier;modified;prev_cosv_file_id
+1;CVE-2022-25296;2021-01-01 00:00:00.000;
+2;CVE-2022-25365;2021-01-01 00:00:00.000;
+3;CVE-2023-22475;2021-01-01 00:00:00.000;
+4;CVE-2022-44697;2021-01-01 00:00:00.000;
+5;CVE-2022-22978;2021-01-01 00:00:00.000;
diff --git a/save-cosv/db/test-data/sqlRequests/lnk-vulnerability-metadata-user.csv b/save-cosv/db/test-data/sqlRequests/lnk-vulnerability-metadata-user.csv
@@ -0,0 +1,9 @@
+id;vulnerability_metadata_id;user_id
+1;1;2
+2;1;3
+3;1;4
+4;1;5
+5;1;6
+6;1;7
+7;2;2
+8;2;3
diff --git a/save-cosv/db/test-data/sqlRequests/vulnerability-metadata-project.csv b/save-cosv/db/test-data/sqlRequests/vulnerability-metadata-project.csv
@@ -0,0 +1,4 @@
+id;name;url;vulnerability_metadata_id;versions;type
+1;"spring-projects/spring-security";"https://github.com/spring-projects/spring-security";1;;PROJECT
+2;"org.springframework.security:spring-security-web";"https://mvnrepository.com/artifact/org.springframework.security/spring-security-web";1;"3.0.0.RELEASE, 3.0.1.RELEASE, 3.0.2.RELEASE, 3.0.3.RELEASE";LIBRARY
+3;"org.springframework.security:spring-security-core";"https://mvnrepository.com/artifact/org.springframework.security/spring-security-core";1;"2.0.0, 2.0.1, 2.0.2, 2.0.3, 2.0.4, 2.0.5.RELEASE";LIBRARY