Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Anonymous IP and Connection Type support #168

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Anonymous IP and Connection Type support #168

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/index.asciidoc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,7 @@ number of cache misses and waste memory.

The path to Maxmind's database file that Logstash should use. The default database is GeoLite2-City.
GeoLite2-City, GeoLite2-Country, GeoLite2-ASN are the free databases from Maxmind that are supported.
GeoIP2-City, GeoIP2-ISP, GeoIP2-Country are the commercial databases from Maxmind that are supported.
GeoIP2-City, GeoIP2-ISP, GeoIP2-Country, GeoIP2-Anonymous-IP, GeoIP2-Connection-Type are the commercial databases from Maxmind that are supported.

If not specified, this will default to the GeoLite2 City database that ships
with Logstash.
Expand Down
6 changes: 6 additions & 0 deletions src/main/java/org/logstash/filters/Fields.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,10 @@
import java.util.Locale;

enum Fields {
ANONYMOUS_TAGS("anonymous_tags"),
AUTONOMOUS_SYSTEM_NUMBER("asn"),
AUTONOMOUS_SYSTEM_ORGANIZATION("as_org"),
CONNECTION_TYPE("connection_type"),
CITY_NAME("city_name"),
COUNTRY_NAME("country_name"),
CONTINENT_CODE("continent_code"),
Expand Down Expand Up @@ -70,6 +72,10 @@ public String fieldName() {
static final EnumSet<Fields> DEFAULT_ASN_LITE_FIELDS = EnumSet.of(Fields.IP, Fields.AUTONOMOUS_SYSTEM_NUMBER,
Fields.AUTONOMOUS_SYSTEM_ORGANIZATION);

static final EnumSet<Fields> DEFAULT_ANONYMOUS_FIELDS = EnumSet.of(Fields.IP, Fields.ANONYMOUS_TAGS);

static final EnumSet<Fields> DEFAULT_CONNECTION_TYPE_FIELDS = EnumSet.of(Fields.IP, Fields.CONNECTION_TYPE);

public static Fields parseField(String value) {
try {
return valueOf(value.toUpperCase(Locale.ROOT));
Expand Down
87 changes: 83 additions & 4 deletions src/main/java/org/logstash/filters/GeoIPFilter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,9 @@
import com.maxmind.geoip2.model.AsnResponse;
import com.maxmind.geoip2.model.CityResponse;
import com.maxmind.geoip2.model.CountryResponse;
import com.maxmind.geoip2.model.ConnectionTypeResponse;
import com.maxmind.geoip2.model.IspResponse;
import com.maxmind.geoip2.model.AnonymousIpResponse;
import com.maxmind.geoip2.record.*;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
Expand Down Expand Up @@ -56,6 +58,15 @@ public class GeoIPFilter {
private static final String CITY_SOUTH_AMERICA_DB_TYPE = "GeoIP2-City-South-America";
private static final String COUNTRY_DB_TYPE = "GeoIP2-Country";
private static final String ISP_DB_TYPE = "GeoIP2-ISP";
private static final String ANONYMOUS_DB_TYPE = "GeoIP2-Anonymous-IP";
private static final String CONNECTION_TYPE_DB_TYPE = "GeoIP2-Connection-Type";

// The Anonymous ip issues
private static final String IS_ANONYMOUS = "is_anonymous";
private static final String IS_ANONYMOUS_VPN = "is_anonymous_vpn";
private static final String IS_HOSTING_PROVIDER = "is_hosting_provider";
private static final String IS_PUBLIC_PROXY = "is_public_proxy";
private static final String IS_TOR_EXIT_NODE = "is_tor_exit_node";

private final String sourceField;
private final String targetField;
Expand Down Expand Up @@ -99,6 +110,12 @@ private Set<Fields> createDesiredFields(List<String> fields) {
case ASN_LITE_DB_TYPE:
desiredFields = Fields.DEFAULT_ASN_LITE_FIELDS;
break;
case ANONYMOUS_DB_TYPE:
desiredFields = Fields.DEFAULT_ANONYMOUS_FIELDS;
break;
case CONNECTION_TYPE_DB_TYPE:
desiredFields = Fields.DEFAULT_CONNECTION_TYPE_FIELDS;
break;
}
} else {
for (String fieldName : fields) {
Expand All @@ -125,7 +142,7 @@ public boolean handleEvent(RubyEvent rubyEvent) {
throw new IllegalArgumentException("Expected input field value to be String or List type");
}

if (ip.trim().isEmpty()){
if (ip.trim().isEmpty()) {
return false;
}

Expand Down Expand Up @@ -153,6 +170,12 @@ public boolean handleEvent(RubyEvent rubyEvent) {
case ISP_DB_TYPE:
geoData = retrieveIspGeoData(ipAddress);
break;
case ANONYMOUS_DB_TYPE:
geoData = retrieveAnonymousData(ipAddress);
break;
case CONNECTION_TYPE_DB_TYPE:
geoData = retrieveConnectionTypeData(ipAddress);
break;
default:
throw new IllegalStateException("Unsupported database type " + databaseReader.getMetadata().getDatabaseType() + "");
}
Expand Down Expand Up @@ -181,13 +204,13 @@ private boolean applyGeoData(Map<String, Object> geoData, Event event) {
}

String s = "[" + this.targetField + "][";
for (Map.Entry<String, Object> it: geoData.entrySet()) {
for (Map.Entry<String, Object> it : geoData.entrySet()) {
event.setField(s + it.getKey() + "]", it.getValue());
}
return true;
}

private Map<String,Object> retrieveCityGeoData(InetAddress ipAddress) throws GeoIp2Exception, IOException {
private Map<String, Object> retrieveCityGeoData(InetAddress ipAddress) throws GeoIp2Exception, IOException {
CityResponse response = databaseReader.city(ipAddress);
Country country = response.getCountry();
City city = response.getCity();
Expand Down Expand Up @@ -302,7 +325,7 @@ private Map<String,Object> retrieveCityGeoData(InetAddress ipAddress) throws Geo
return geoData;
}

private Map<String,Object> retrieveCountryGeoData(InetAddress ipAddress) throws GeoIp2Exception, IOException {
private Map<String, Object> retrieveCountryGeoData(InetAddress ipAddress) throws GeoIp2Exception, IOException {
CountryResponse response = databaseReader.country(ipAddress);
Country country = response.getCountry();
Continent continent = response.getContinent();
Expand Down Expand Up @@ -401,4 +424,60 @@ private Map<String, Object> retrieveAsnGeoData(InetAddress ipAddress) throws Geo

return geoData;
}

private Map<String, Object> retrieveAnonymousData(InetAddress ipAddress) throws GeoIp2Exception, IOException {
AnonymousIpResponse response = databaseReader.anonymousIp(ipAddress);
Map<String, Object> geoData = new HashMap<>();
List<String> anonymousTags = new ArrayList<>();
for (Fields desiredField : this.desiredFields) {
switch (desiredField) {
case IP:
geoData.put(Fields.IP.fieldName(), ipAddress.getHostAddress());
break;
case ANONYMOUS_TAGS:
if(response.isAnonymous()) {
anonymousTags.add(GeoIPFilter.IS_ANONYMOUS);
}
if(response.isAnonymousVpn()) {
anonymousTags.add(GeoIPFilter.IS_ANONYMOUS_VPN);
}
if(response.isHostingProvider()) {
anonymousTags.add(GeoIPFilter.IS_HOSTING_PROVIDER);
}
if(response.isPublicProxy()) {
anonymousTags.add(GeoIPFilter.IS_PUBLIC_PROXY);
}
if(response.isTorExitNode()) {
anonymousTags.add(GeoIPFilter.IS_TOR_EXIT_NODE);
}
break;
}
}
// add anonymmous tags field if there is at least one issue
if(!anonymousTags.isEmpty()){
geoData.put(Fields.ANONYMOUS_TAGS.fieldName(), anonymousTags);
}

return geoData;
}

private Map<String, Object> retrieveConnectionTypeData(InetAddress ipAddress) throws GeoIp2Exception, IOException {
ConnectionTypeResponse response = databaseReader.connectionType(ipAddress);
Map<String, Object> geoData = new HashMap<>();
for (Fields desiredField : this.desiredFields) {
switch (desiredField) {
case IP:
geoData.put(Fields.IP.fieldName(), ipAddress.getHostAddress());
break;
case CONNECTION_TYPE:
ConnectionTypeResponse.ConnectionType connectionType = response.getConnectionType();
if (connectionType != null) {
geoData.put(Fields.CONNECTION_TYPE.fieldName(), connectionType.name());
}
break;
}
}

return geoData;
}
}