Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactor tests #157

Open
wants to merge 46 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 42 commits
Commits
Show all changes
46 commits
Select commit Hold shift + click to select a range
1e384e8
initial commit
jdr0887 Oct 24, 2022
5f7c08e
initial commit
jdr0887 Oct 24, 2022
7cca582
moving validate to util
jdr0887 Nov 7, 2022
a9b478a
refactor
jdr0887 Nov 11, 2022
6b2a371
removing
jdr0887 Nov 29, 2022
ba91101
removing
jdr0887 Nov 29, 2022
64a4949
initial commit
jdr0887 Nov 29, 2022
980846c
adding independent log config
jdr0887 Nov 29, 2022
190efbd
posterity
jdr0887 Nov 29, 2022
f1073eb
initial commit
jdr0887 Nov 29, 2022
7fe4490
formatting
jdr0887 Nov 29, 2022
0b3a56e
formatting
jdr0887 Nov 29, 2022
11e589d
adding redis databases to match yaml config
jdr0887 Nov 29, 2022
15068f2
formatting, changing gene_protein_db to conflation_db
jdr0887 Nov 29, 2022
507e372
renaming variables
jdr0887 Nov 29, 2022
a6be46c
reducing size
jdr0887 Nov 29, 2022
20a6b98
adding tests from test_norm.py
jdr0887 Nov 29, 2022
f07527a
formatting, fixing renamed service
jdr0887 Nov 29, 2022
e820ad5
adding conflation_type
jdr0887 Nov 29, 2022
2d8e935
adding asyncclick
jdr0887 Nov 29, 2022
00c2591
adding validate_compendium
jdr0887 Nov 29, 2022
2a95ccc
adding conflation enum, formatting
jdr0887 Nov 29, 2022
c06a4e3
formatting
jdr0887 Nov 29, 2022
3614cd4
formatting, adding conflation type
jdr0887 Nov 29, 2022
ba7630a
adding conflation type
jdr0887 Nov 29, 2022
cc21083
adding conflation type
jdr0887 Nov 29, 2022
fb44db7
adding more logging statements
jdr0887 Nov 29, 2022
df9aece
moving validate to util.py
jdr0887 Nov 29, 2022
4cc6fa6
removing
jdr0887 Nov 29, 2022
f268092
removing config & load.py
jdr0887 Nov 29, 2022
7daeeac
removing config.json, redis_config.yaml, and load.py
jdr0887 Nov 29, 2022
3d6b00c
adding redis instances to match redis_config.yaml
jdr0887 Nov 29, 2022
137e936
renaming
jdr0887 Nov 29, 2022
3ede74a
initial commit
jdr0887 Nov 29, 2022
90dfe3b
setting build & pull to true
jdr0887 Nov 29, 2022
dfeca95
removing unused env vars
jdr0887 Nov 29, 2022
1c0c689
initial commit
jdr0887 Nov 30, 2022
55ae81a
initial commit
jdr0887 Nov 30, 2022
b98b1a3
ignoring Cargo lock
jdr0887 Nov 30, 2022
bf78fab
conflation file has new format
jdr0887 Nov 30, 2022
372ebf7
adding get_semantic_types test
jdr0887 Nov 30, 2022
2e56b37
renamed
jdr0887 Nov 30, 2022
0d8815d
close too many on shutdown event
jdr0887 Dec 1, 2022
1f859e1
updating
jdr0887 Dec 1, 2022
8130347
checking for gene_protein
jdr0887 Dec 1, 2022
a48c817
moving contents of readme to wiki page
jdr0887 Dec 1, 2022
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -130,3 +130,5 @@ dmypy.json

# intellij
.idea

Cargo.lock
2 changes: 0 additions & 2 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,7 @@ WORKDIR /code
COPY ./requirements.txt requirements.txt
COPY ./setup.py setup.py
COPY ./node_normalizer node_normalizer
COPY ./config.json config.json
COPY ./redis_config.yaml redis_config.yaml
COPY ./load.py load.py

# install requirements
RUN pip install -r requirements.txt
Expand Down
4 changes: 1 addition & 3 deletions Dockerfile-test
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,7 @@ WORKDIR /code
COPY ./requirements.txt requirements.txt
COPY ./setup.py setup.py
COPY ./node_normalizer node_normalizer
COPY ./tests/config.json config.json
COPY ./tests/redis_config.yaml redis_config.yaml
COPY ./load.py load.py
COPY ./redis_config.yaml redis_config.yaml
COPY ./tests tests

# install requirements
Expand Down
20 changes: 0 additions & 20 deletions KGX_converter.py

This file was deleted.

17 changes: 0 additions & 17 deletions config.json

This file was deleted.

78 changes: 49 additions & 29 deletions docker-compose-test.yml
Original file line number Diff line number Diff line change
@@ -1,34 +1,41 @@
version: "3"
services:

r3:
container_name: r3
image: r3
build:
context: .
dockerfile: Dockerfile-test
env_file: .env
redis_eq_id_to_id_db:
container_name: redis_eq_id_to_id_db
image: redis:alpine
expose:
- 8080
ports:
- "8080:8080"
# links:
# - callback-app
# networks:
# - nn_network
depends_on:
- redis
- callback-app
- 6379

redis:
container_name: redis
redis_id_to_eqids_db:
container_name: redis_id_to_eqids_db
image: redis:alpine
expose:
- 6379

redis_id_to_type_db:
container_name: redis_id_to_type_db
image: redis:alpine
expose:
- 6379

redis_curie_to_bl_type_db:
container_name: redis_curie_to_bl_type_db
image: redis:alpine
expose:
- 6379

redis_info_content_db:
container_name: redis_info_content_db
image: redis:alpine
expose:
- 6379

redis_conflation_db:
container_name: redis_conflation_db
image: redis:alpine
ports:
- "6379:6379"
expose:
- 6379
# networks:
# - nn_network

callback-app:
container_name: callback-app
Expand All @@ -39,10 +46,23 @@ services:
- 8008
ports:
- "8008:8008"
# networks:
# - nn_network

#networks:
# nn_network:
# name: nn
# external: false
node-norm:
container_name: node-norm
image: node-norm
build:
context: .
dockerfile: Dockerfile-test
env_file: .env
expose:
- 8080
ports:
- "8080:8080"
depends_on:
- redis_eq_id_to_id_db
- redis_id_to_eqids_db
- redis_id_to_type_db
- redis_curie_to_bl_type_db
- redis_info_content_db
- redis_conflation_db
- callback-app
59 changes: 49 additions & 10 deletions docker-compose.yml
Original file line number Diff line number Diff line change
@@ -1,18 +1,57 @@
version: "3"
services:
redis:
container_name: redis
redis_eq_id_to_id_db:
container_name: redis_eq_id_to_id_db
image: redis:alpine
expose:
- 6379

r3:
container_name: r3
image: r3
build: .
env_file:
- .env

redis_id_to_eqids_db:
container_name: redis_id_to_eqids_db
image: redis:alpine
expose:
- 6379

redis_id_to_type_db:
container_name: redis_id_to_type_db
image: redis:alpine
expose:
- 6379

redis_curie_to_bl_type_db:
container_name: redis_curie_to_bl_type_db
image: redis:alpine
expose:
- 6379

redis_info_content_db:
container_name: redis_info_content_db
image: redis:alpine
expose:
- 6379

redis_conflation_db:
container_name: redis_conflation_db
image: redis:alpine
expose:
- 6379

node-norm:
container_name: node-norm
image: node-norm
build:
context: .
dockerfile: Dockerfile
env_file: .env
expose:
- 8080
ports:
- "8080:8080"
depends_on:
- redis
- redis_eq_id_to_id_db
- redis_id_to_eqids_db
- redis_id_to_type_db
- redis_curie_to_bl_type_db
- redis_info_content_db
- redis_conflation_db

20 changes: 0 additions & 20 deletions load.py

This file was deleted.

21 changes: 21 additions & 0 deletions nn_io_rs/Cargo.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
[package]
name = "nn_io_rs"
version = "0.1.0"
edition = "2021"

# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[lib]
name = "nn_io_rs"
crate-type = ["cdylib"]

[dependencies]
clap = { version = "^4.0", features = ["derive"] }
csv = "^1.1"
humantime = "^2.1"
itertools = "^0.10"
pyo3 = { version = "^0.17", features = ["extension-module"] }
reqwest = { version = "^0.11", features = ["json"] }
tokio = { version = "^1", features = ["full"] }
serde = "^1.0"
serde_derive = "^1.0"
serde_json = "^1.0"
4 changes: 4 additions & 0 deletions nn_io_rs/rustfmt.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
max_width = 180
newline_style = "Unix"
use_field_init_shorthand = true
use_try_shorthand = true
68 changes: 68 additions & 0 deletions nn_io_rs/src/bin/pull_and_filter_compendia_data.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
use clap::Parser;
use reqwest::StatusCode;
use serde_json::Value;
use std::error;
use std::fs::File;
use std::io::{BufWriter, Write};
use std::path;

#[derive(Parser, PartialEq, Debug)]
#[command(author, version, about, long_about = None)]
struct Options {
#[arg(short, long, default_value_t = 3_000)]
entry_limit: usize,

#[arg(short, long, long_help = "identifiers greater than", default_value_t = 2)]
identifier_floor: usize,

#[arg(short, long)]
output_dir: path::PathBuf,

#[arg(short, long)]
file_name: Vec<String>,
}
#[tokio::main]
async fn main() -> Result<(), Box<dyn error::Error>> {
let options = Options::parse();

let output_dir: path::PathBuf = options.output_dir.to_path_buf();
std::fs::create_dir_all(&output_dir).ok();

for file_name in options.file_name.iter() {
let mut output: path::PathBuf = output_dir.clone();
output.push(file_name);

let file = File::create(output.as_path()).unwrap();
let mut writer = BufWriter::new(file);

let cell_data_response = reqwest::get(format!("https://stars.renci.org/var/babel_outputs/2022oct13/compendia/{}", file_name)).await?;

match cell_data_response.status() {
StatusCode::OK => {
let content = cell_data_response.text().await?;
let mut cache = vec![];
for line in content.lines() {
let json: Value = serde_json::from_str(line).expect("could not parse line to json");
if json["identifiers"].is_array() {
let identifiers = json["identifiers"].as_array().expect("could not convert to array");
if identifiers.len() >= options.identifier_floor {
cache.push(line);
}
}
if cache.len() == options.entry_limit {
break;
}
}

cache.iter().for_each(|line| {
writer.write(format!("{}\n", line).as_bytes()).expect("Unable to write data");
});
}
_ => {
print!("response status was not OK (200)");
}
}
}

Ok(())
}
Loading