Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Greenbids fix geolookup: host and fetch GCS + mock dbReader UT #3626

Open
wants to merge 15 commits into
base: master
Choose a base branch
from
Open
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
package org.prebid.server.hooks.modules.greenbids.real.time.data.config;

import com.google.cloud.storage.Blob;
import com.google.cloud.storage.Storage;
import com.google.cloud.storage.StorageException;
import io.vertx.core.Promise;
import io.vertx.core.Vertx;
import com.maxmind.geoip2.DatabaseReader;
Expand All @@ -8,36 +11,39 @@

import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Optional;
import java.util.concurrent.atomic.AtomicReference;

public class DatabaseReaderFactory implements Initializable {

private final String geoLiteCountryUrl;
private final String gcsBucketName;

private final String geoLiteCountryPath;

private final Vertx vertx;

private final Storage storage;

private final AtomicReference<DatabaseReader> databaseReaderRef = new AtomicReference<>();

public DatabaseReaderFactory(String geoLitCountryUrl, Vertx vertx) {
this.geoLiteCountryUrl = geoLitCountryUrl;
public DatabaseReaderFactory(String gcsBucketName, String geoLiteCountryPath, Vertx vertx, Storage storage) {
this.gcsBucketName = gcsBucketName;
this.geoLiteCountryPath = geoLiteCountryPath;
this.vertx = vertx;
this.storage = storage;
}

@Override
public void initialize(Promise<Void> initializePromise) {

vertx.executeBlocking(() -> {
try {
final URL url = new URL(geoLiteCountryUrl);
final Blob blob = getBlob();
final Path databasePath = Files.createTempFile("GeoLite2-Country", ".mmdb");

try (InputStream inputStream = url.openStream();
FileOutputStream outputStream = new FileOutputStream(databasePath.toFile())) {
inputStream.transferTo(outputStream);
try (FileOutputStream outputStream = new FileOutputStream(databasePath.toFile())) {
outputStream.write(blob.getContent());
}

databaseReaderRef.set(new DatabaseReader.Builder(databasePath.toFile()).build());
Expand All @@ -49,6 +55,16 @@ public void initialize(Promise<Void> initializePromise) {
.onComplete(initializePromise);
}

private Blob getBlob() {
try {
return Optional.ofNullable(storage.get(gcsBucketName))
.map(bucket -> bucket.get(geoLiteCountryPath))
.orElseThrow(() -> new PreBidException("Bucket not found: " + gcsBucketName));
} catch (StorageException e) {
throw new PreBidException("Error accessing GCS artefact for model: ", e);
}
}

public DatabaseReader getDatabaseReader() {
return databaseReaderRef.get();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,10 @@
public class GreenbidsRealTimeDataConfiguration {

@Bean
DatabaseReaderFactory databaseReaderFactory(GreenbidsRealTimeDataProperties properties, Vertx vertx) {
return new DatabaseReaderFactory(properties.getGeoLiteCountryPath(), vertx);
DatabaseReaderFactory databaseReaderFactory(
GreenbidsRealTimeDataProperties properties, Vertx vertx, Storage storage) {
return new DatabaseReaderFactory(
properties.getGcsBucketName(), properties.getGeoLiteCountryPath(), vertx, storage);
}

@Bean
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.iab.openrtb.request.BidRequest;
import com.iab.openrtb.request.Device;
import com.iab.openrtb.request.Geo;
import com.iab.openrtb.request.Imp;
import com.maxmind.geoip2.DatabaseReader;
import com.maxmind.geoip2.exception.GeoIp2Exception;
Expand Down Expand Up @@ -86,12 +87,16 @@ private List<ThrottlingMessage> extractMessagesForImp(
final String ip = Optional.ofNullable(bidRequest.getDevice())
.map(Device::getIp)
.orElse(null);
final String countryFromIp = getCountry(ip);
final String country = Optional.ofNullable(bidRequest.getDevice())
.map(Device::getGeo)
.map(Geo::getCountry)
.orElseGet(() -> getCountry(ip));

return createThrottlingMessages(
bidderNode,
impId,
greenbidsUserAgent,
countryFromIp,
country,
hostname,
hourBucket,
minuteQuadrant);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,12 @@
import com.iab.openrtb.request.Device;
import com.iab.openrtb.request.Imp;
import com.maxmind.geoip2.DatabaseReader;
import com.maxmind.geoip2.exception.GeoIp2Exception;
import com.maxmind.geoip2.model.CountryResponse;
import com.maxmind.geoip2.record.Country;
import io.vertx.core.Future;
import io.vertx.core.Vertx;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
Expand Down Expand Up @@ -52,12 +54,9 @@
import org.prebid.server.proto.openrtb.ext.request.ExtRequest;
import org.prebid.server.proto.openrtb.ext.request.ExtRequestPrebid;

import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.net.InetAddress;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Collections;
import java.util.List;
Expand All @@ -66,6 +65,7 @@

import static java.util.function.UnaryOperator.identity;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mock.Strictness.LENIENT;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
Expand All @@ -88,15 +88,27 @@ public class GreenbidsRealTimeDataProcessedAuctionRequestHookTest {
@Mock(strictness = LENIENT)
private DatabaseReaderFactory databaseReaderFactory;

@Mock
private DatabaseReader dbReader;
@Mock(strictness = LENIENT)
private DatabaseReader databaseReader;

@Mock(strictness = LENIENT)
private CountryResponse countryResponse;

@Mock(strictness = LENIENT)
private Country country;

private GreenbidsRealTimeDataProcessedAuctionRequestHook target;

@BeforeEach
public void setUp() throws IOException {
public void setUp() throws IOException, GeoIp2Exception {
final Storage storage = StorageOptions.newBuilder()
.setProjectId("test_project").build().getService();

when(country.getName()).thenReturn("United States");
when(countryResponse.getCountry()).thenReturn(country);
when(databaseReader.country(any(InetAddress.class))).thenReturn(countryResponse);
when(databaseReaderFactory.getDatabaseReader()).thenReturn(databaseReader);

final FilterService filterService = new FilterService();
final OnnxModelRunnerFactory onnxModelRunnerFactory = new OnnxModelRunnerFactory();
final ThrottlingThresholdsFactory throttlingThresholdsFactory = new ThrottlingThresholdsFactory();
Expand All @@ -118,7 +130,7 @@ public void setUp() throws IOException {
final OnnxModelRunnerWithThresholds onnxModelRunnerWithThresholds = new OnnxModelRunnerWithThresholds(
modelCache,
thresholdCache);
when(databaseReaderFactory.getDatabaseReader()).thenReturn(dbReader);

final GreenbidsInferenceDataService greenbidsInferenceDataService = new GreenbidsInferenceDataService(
databaseReaderFactory,
TestBidRequestProvider.MAPPER);
Expand Down Expand Up @@ -162,7 +174,6 @@ public void callShouldExitEarlyWhenPartnerNotActivatedInBidRequest() {
assertThat(result.analyticsTags()).isNull();
}

@Disabled("Broken until dbReader is mocked")
@Test
public void callShouldNotFilterBiddersAndReturnAnalyticsTagWhenExploration() throws OrtException, IOException {
// given
Expand Down Expand Up @@ -217,7 +228,6 @@ public void callShouldNotFilterBiddersAndReturnAnalyticsTagWhenExploration() thr
assertThat(fingerprint).isNotNull();
}

@Disabled("Broken until dbReader is mocked")
@Test
public void callShouldFilterBiddersBasedOnModelWhenAnyFeatureNotAvailable() throws OrtException, IOException {
// given
Expand Down Expand Up @@ -278,7 +288,6 @@ public void callShouldFilterBiddersBasedOnModelWhenAnyFeatureNotAvailable() thro
assertThat(resultBidRequest).usingRecursiveComparison().isEqualTo(expectedBidRequest);
}

@Disabled("Broken until dbReader is mocked")
@Test
public void callShouldFilterBiddersBasedOnModelResults() throws OrtException, IOException {
// given
Expand Down
Loading