Add G groups, frequency data, and update protein sequence to

Makefile

@@ -48,7 +48,7 @@ LIB = lib
 ROBOT := java -jar build/robot.jar
 TODAY := $(shell date +%Y-%m-%d)
 
-tables = external core genetic-locus haplotype serotype chain molecule haplotype-molecule serotype-molecule mutant-molecule evidence chain-sequence
+tables = external core genetic-locus haplotype serotype chain molecule haplotype-molecule serotype-molecule mutant-molecule evidence chain-sequence gene-allele G-group gene-alleles properties chain-frequencies G-group-frequencies gene-allele-frequencies gene-allele
 source_files = $(foreach o,$(tables),ontology/$(o).tsv)
 build_files = $(foreach o,$(tables),build/$(o).tsv)
 templates = $(foreach i,$(build_files),--template $(i))
@@ -99,7 +99,7 @@ load: $(COGS_SHEETS)
 push: | .cogs
 	cogs push
 
-.PHONY: destroy 
+.PHONY: destroy
 destroy: | .cogs
 	cogs delete -f
 
@@ -177,7 +177,7 @@ build/mutant-molecule.tsv: src/scripts/synonyms.py build/mutant-molecule-fixed.t
 	python3 $^ > $@
 
 # Represent tables in Excel
-mro.xlsx: src/scripts/tsv2xlsx.py index.tsv iedb/iedb.tsv ontology/genetic-locus.tsv ontology/haplotype.tsv ontology/serotype.tsv ontology/chain.tsv ontology/chain-sequence.tsv ontology/molecule.tsv ontology/haplotype-molecule.tsv ontology/serotype-molecule.tsv ontology/mutant-molecule.tsv ontology/core.tsv ontology/external.tsv iedb/iedb-manual.tsv ontology/evidence.tsv ontology/rejected.tsv
+mro.xlsx: src/scripts/tsv2xlsx.py index.tsv iedb/iedb.tsv ontology/genetic-locus.tsv ontology/haplotype.tsv ontology/serotype.tsv ontology/chain.tsv ontology/chain-sequence.tsv ontology/molecule.tsv ontology/haplotype-molecule.tsv ontology/serotype-molecule.tsv ontology/mutant-molecule.tsv ontology/core.tsv ontology/external.tsv iedb/iedb-manual.tsv ontology/evidence.tsv ontology/rejected.tsv ontology/G-group.tsv ontology/gene-allele.tsv ontology/gene-alleles.tsv ontology/properties.tsv ontology/chain-frequencies.tsv ontology/G-group-frequencies.tsv ontology/gene-allele-frequencies.tsv
 	python3 $< $@ $(wordlist 2,100,$^)
 
 update-tsv: update-tsv-files sort build/whitespace.tsv
@@ -201,6 +201,8 @@ sort:
 build/whitespace.tsv: src/scripts/validation/detect_whitespace.py index.tsv iedb/iedb.tsv iedb/iedb-manual.tsv $(source_files)
 	python3 $^ $@
 
+build/HLA-%-frequency.xlsx: | build
+	curl -o $@ -L "https://s3.eu-central-1.amazonaws.com/ihiw.website.data/CIWD-3.0/HLA-$*_PrimaryData-IHWS-20200320.xlsx"
 
 ### Sequences
 
@@ -212,6 +214,15 @@ build/hla.fasta: | build
 build/mhc.fasta: | build
 	curl -L -o $@ ftp://ftp.ebi.ac.uk/pub/databases/ipd/mhc/MHC_prot.fasta
 
+build/hla.dat: | build
+	curl -o $@ -L https://github.com/ANHIG/IMGTHLA/raw/Latest/hla.dat
+
+build/hla1.dat: | build
+	curl -o $@ -L https://raw.githubusercontent.com/ANHIG/IMGTHLA/3310/hla.dat
+
+build/hla_nom_g.txt: | build
+	curl -o $@ -L https://github.com/ANHIG/IMGTHLA/raw/Latest/wmda/hla_nom_g.txt
+
 # update-seqs will only write seqs to terms without seqs
 .PHONY: update-seqs
 update-seqs: src/scripts/update_seqs.py ontology/chain-sequence.tsv build/hla.fasta build/mhc.fasta
@@ -228,6 +239,17 @@ build/hla_prot.fasta: | build
 build/AlleleList.txt: | build
 	curl -o $@ -L https://raw.githubusercontent.com/ANHIG/IMGTHLA/Latest/Allelelist.txt
 
+src/dbfetch.py:
+	curl -o $@ -L https://raw.githubusercontent.com/ebi-wp/webservice-clients/master/python/dbfetch.py
+
+.PHONY: update-G-groups
+update-G-groups: build/hla.dat build/hla_nom_g.txt ontology/chain-sequence.tsv
+		python3 src/scripts/alleles/update_gene_allele_seq.py -u
+
+.PHONY: add-frequency-data
+add-frequency-data: src/dbfetch.py build/hla1.dat ontology/G-group.tsv ontology/gene-alleles.tsv build/report-g-grp.json build/HLA-A-frequency.xlsx build/HLA-B-frequency.xlsx build/HLA-C-frequency.xlsx build/HLA-DRB1-frequency.xlsx build/HLA-DRB3-frequency.xlsx build/HLA-DRB4-frequency.xlsx build/HLA-DRB5-frequency.xlsx build/HLA-DQB1-frequency.xlsx build/HLA-DPB1-frequency.xlsx
+	python3 src/scripts/alleles/update_gene_allele_seq.py -f
+
 .PHONY: update-alleles
 update-alleles: src/scripts/alleles/update_human_alleles.py ontology/chain-sequence.tsv ontology/chain.tsv ontology/molecule.tsv ontology/genetic-locus.tsv index.tsv build/hla_prot.fasta build/AlleleList.txt
 	python3 $^
@@ -250,12 +272,12 @@ update-sla-alleles: src/scripts/alleles/update_sla_alleles.py ontology/chain-seq
 
 
 ### OWL Files
-
 mro.owl: build/mro-import.owl index.tsv $(build_files) ontology/metadata.ttl | build/robot.jar
 	$(ROBOT) template \
 	--input $< \
 	--prefix "MRO: $(OBO)/MRO_" \
 	--prefix "REO: $(OBO)/REO_" \
+	--prefix "NCIT: $(OBO)/NCIT_" \
 	--template index.tsv \
 	$(templates) \
 	--merge-before \
@@ -269,12 +291,13 @@ mro.owl: build/mro-import.owl index.tsv $(build_files) ontology/metadata.ttl | b
 	--annotation-file ontology/metadata.ttl \
 	--output $@
 
-build/mro-import.owl: build/eco-import.ttl build/iao-import.ttl build/obi-import.ttl build/ro-import.ttl ontology/import.txt | build/robot.jar
+build/mro-import.owl: build/eco-import.ttl build/iao-import.ttl build/obi-import.ttl build/ro-import.ttl build/hancestro-import.ttl ontology/import.txt | build/robot.jar
 	$(ROBOT) merge \
 	--input build/eco-import.ttl \
 	--input build/obi-import.ttl \
 	--input build/ro-import.ttl \
 	--input build/iao-import.ttl \
+	--input build/hancestro-import.ttl \
 	extract \
 	--method MIREOT \
 	--upper-term "GO:0008150" \
@@ -283,14 +306,19 @@ build/mro-import.owl: build/eco-import.ttl build/iao-import.ttl build/obi-import
 	--upper-term "ECO:0000000" \
 	--upper-term "BFO:0000040" \
 	--upper-term "PR:000000001" \
-	--lower-terms $(word 5,$^) \
+	--lower-terms $(word 6,$^) \
 	--output $@
 
 # fetch ontology dependencies
 $(LIB)/%:
 	mkdir -p $(LIB)
 	cd $(LIB) && curl -LO "$(OBO)/$*"
 
+$(LIB)/ro.owl: build/robot.jar
+	mkdir -p $(LIB)
+	cd $(LIB) && curl -LO "$(OBO)/ro.owl"
+	$(ROBOT) merge --input $@ --output $@
+
 UC = $(shell echo '$1' | tr '[:lower:]' '[:upper:]')
 
 # OBI IAO:0000115 has mulitples so get the definiton from here
@@ -301,7 +329,7 @@ build/%.txt: ontology/import.txt | build
 # RO:0000056 isn't in RO?
 # we could also just add this to index.tsv
 build/obi.txt: ontology/import.txt | build
-	sed '/^ECO/d' $< | sed '/^RO/d' | sed '/^IAO/d' > $@
+	sed '/^ECO/d' $< | sed '/^RO/d' | sed '/^IAO/d' | sed '/^HANCESTRO/d' | sed '/^GSSO/d' | sed '/^NCIT/d' | sed '/^IDO/d' > $@
 	echo "RO:0000056" >> $@
 
 build/%.db: src/queries/prefixes.sql $(LIB)/%.owl | build/rdftab
@@ -439,7 +467,6 @@ prepare: update-seqs
 prepare: update-iedb
 prepare:
 	pip install -r requirements.txt
-
 .PHONY: clean
 clean:
 	rm -rf mro.owl