diff --git a/IndexSearchService/README.md b/IndexSearchService/README.md deleted file mode 100644 index a59f6a5..0000000 --- a/IndexSearchService/README.md +++ /dev/null @@ -1,11 +0,0 @@ -# IndexSearchService -1. How to use Elasticsearch. -2. How to use Solr or Lucene to index and query data. -3. How to use Lucene to index and query POI(GEO data). - -中文详情（Chinese Details）： -* [Elasticsearch index and query data](http://www.cnblogs.com/luxiaoxun/p/4869509.html) -* [SQL to Elasticsearch Java Code](http://www.cnblogs.com/luxiaoxun/p/6826211.html) -* [Lucene index and query POI data](http://www.cnblogs.com/luxiaoxun/p/5020247.html) -* [Solr index and query MYSQL data](http://www.cnblogs.com/luxiaoxun/p/4442770.html) -* [Solr index and query GEO data](http://www.cnblogs.com/luxiaoxun/p/4477591.html) diff --git a/IndexSearchService/lib/lucene-analyzers-ik-1.0.jar b/IndexSearchService/lib/lucene-analyzers-ik-1.0.jar deleted file mode 100644 index d4ff268..0000000 Binary files a/IndexSearchService/lib/lucene-analyzers-ik-1.0.jar and /dev/null differ diff --git a/IndexSearchService/pom.xml b/IndexSearchService/pom.xml deleted file mode 100644 index f0be9d5..0000000 --- a/IndexSearchService/pom.xml +++ /dev/null @@ -1,214 +0,0 @@ - - - Code4Java - com.luxx - 1.0.0 - - 4.0.0 - - IndexSearchService - jar - - - UTF-8 - 2.2.7.RELEASE - - 7.2.1 - 6.2.4 - 2.12.1 - 2.17.0 - 3.4.5 - 8.0.16 - - - - - redshift - http://redshift-maven-repository.s3-website-us-east-1.amazonaws.com/release - - - - - - org.wltea.analyzer - lucene-analyzers-ik - 1.0 - system - ${project.basedir}/lib/lucene-analyzers-ik-1.0.jar - - - - - org.springframework.boot - spring-boot-starter - ${spring-boot.version} - - - org.springframework.boot - spring-boot-starter-logging - - - - - org.springframework.boot - spring-boot-starter-log4j2 - ${spring-boot.version} - - - - - net.minidev - json-smart - 2.4.7 - - - - - com.fasterxml.jackson.core - jackson-core - ${jackson.version} - - - com.fasterxml.jackson.core - jackson-databind - ${jackson.version} - - - - mysql - mysql-connector-java - ${mysql.version} - - - - com.zaxxer - HikariCP - ${hikaricp.version} - - - - org.elasticsearch - elasticsearch - ${es.version} - - - org.elasticsearch.client - transport - ${es.version} - - - org.elasticsearch.client - elasticsearch-rest-client - ${es.version} - - - org.elasticsearch.client - elasticsearch-rest-high-level-client - ${es.version} - - - - - org.apache.logging.log4j - log4j-api - ${log4j2.version} - - - org.apache.logging.log4j - log4j-core - ${log4j2.version} - - - - com.amazon.redshift - redshift-jdbc42 - 1.2.1.1001 - - - - - org.apache.lucene - lucene-core - ${lucene.version} - - - org.apache.lucene - lucene-queries - ${lucene.version} - - - org.apache.lucene - lucene-queryparser - ${lucene.version} - - - org.apache.lucene - lucene-analyzers-common - ${lucene.version} - - - org.apache.lucene - lucene-analyzers-smartcn - ${lucene.version} - - - org.apache.lucene - lucene-spatial - ${lucene.version} - - - - - org.locationtech.spatial4j - spatial4j - 0.7 - - - - - org.apache.httpcomponents - httpclient - 4.5.13 - - - org.apache.httpcomponents - httpcore - 4.4.1 - - - org.apache.httpcomponents - httpmime - 4.4.1 - - - - - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.8.1 - - 8 - 8 - - - - - org.springframework.boot - spring-boot-maven-plugin - ${spring-boot.version} - - - - repackage - - - - - - - - diff --git a/IndexSearchService/src/main/java/com/luxx/index/DataIndexApp.java b/IndexSearchService/src/main/java/com/luxx/index/DataIndexApp.java deleted file mode 100644 index 99fb5ab..0000000 --- a/IndexSearchService/src/main/java/com/luxx/index/DataIndexApp.java +++ /dev/null @@ -1,15 +0,0 @@ -package com.luxx.index; - -import org.springframework.boot.SpringApplication; -import org.springframework.boot.SpringBootConfiguration; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.context.annotation.ComponentScan; - -@SpringBootApplication -@SpringBootConfiguration -@ComponentScan(basePackages = "com.luxx.index") -public class DataIndexApp { - public static void main(String[] args) { - SpringApplication.run(DataIndexApp.class, args); - } -} diff --git a/IndexSearchService/src/main/java/com/luxx/index/client/ElasticSearchClient.java b/IndexSearchService/src/main/java/com/luxx/index/client/ElasticSearchClient.java deleted file mode 100644 index ff5f5b8..0000000 --- a/IndexSearchService/src/main/java/com/luxx/index/client/ElasticSearchClient.java +++ /dev/null @@ -1,260 +0,0 @@ -package com.luxx.index.client; - -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; -import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; -import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest; -import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; -import org.elasticsearch.action.bulk.BulkRequestBuilder; -import org.elasticsearch.action.index.IndexRequestBuilder; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.Requests; -import org.elasticsearch.client.transport.TransportClient; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.cluster.metadata.MappingMetaData; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.TransportAddress; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.SearchHits; -import org.elasticsearch.search.aggregations.AggregationBuilder; -import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder; -import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; -import org.elasticsearch.search.aggregations.bucket.histogram.ParsedDateHistogram; -import org.elasticsearch.search.aggregations.bucket.terms.Terms; -import org.elasticsearch.search.aggregations.metrics.max.Max; -import org.elasticsearch.search.aggregations.metrics.sum.Sum; -import org.elasticsearch.transport.client.PreBuiltTransportClient; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.context.annotation.Lazy; -import org.springframework.stereotype.Component; - -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; - -@Component -@Lazy -@ConditionalOnProperty(name = "index.es.cluster.address") -public class ElasticSearchClient { - private static Logger log = LogManager.getLogger(ElasticSearchClient.class); - - @Value("${index.es.cluster.address}") - private String esAddress; - - @Value("${index.es.cluster.name}") - private String esName; - - // ES Client - private TransportClient client; - - @PostConstruct - public void init() throws UnknownHostException { - log.info("es.cluster.name: " + esName); - log.info("es.cluster.address: " + esAddress); - - Settings settings = Settings.builder() - .put("cluster.name", esName) - .put("client.transport.sniff", false).build(); - client = new PreBuiltTransportClient(settings); - for (String address : esAddress.split(",")) { - String[] hostPort = address.split(":"); - client.addTransportAddress(new TransportAddress(InetAddress.getByName(hostPort[0]), - Integer.parseInt(hostPort[1]))); - } - } - - @PreDestroy - public void close() { - if (client != null) { - client.close(); - } - } - - // Create Index - public void createIndex(String index) { - IndicesExistsResponse indicesExistsResponse = client.admin().indices() - .exists(new IndicesExistsRequest(new String[]{index})).actionGet(); - if (!indicesExistsResponse.isExists()) { - client.admin().indices().create(new CreateIndexRequest(index)).actionGet(); - } - } - - // Delete Index - public void deleteIndex(String index) { - IndicesExistsResponse indicesExistsResponse = client.admin().indices() - .exists(new IndicesExistsRequest(new String[]{index})).actionGet(); - if (indicesExistsResponse.isExists()) { - client.admin().indices().delete(new DeleteIndexRequest(index)).actionGet(); - } - } - - // Delete Type - public void deleteType(String index, String type) { - client.prepareDelete().setIndex(index).setType(type).execute().actionGet(); - } - - private boolean isMappingExist(String index, String type) { - ClusterState cs = client.admin().cluster().prepareState().setIndices(index).execute().actionGet().getState(); - // Check index metadata existence. - IndexMetaData imd = cs.getMetaData().index(index); - if (imd == null) { - return false; - } - // Check mapping metadata existence. - MappingMetaData mdd = imd.mapping(type); - if (mdd != null) { - return true; - } - return false; - } - - // Define type mapping - public void defineIndexTypeMapping(String index, String type, XContentBuilder mapBuilder) { - if (!isMappingExist(index, type)) { - PutMappingRequest putMappingRequest = Requests.putMappingRequest(index).type(type).source(mapBuilder); - client.admin().indices().putMapping(putMappingRequest).actionGet(); - } - } - - public BulkRequestBuilder getBulkRequest() { - return client.prepareBulk(); - } - - public IndexRequestBuilder getIndexRequest(String index, String type, String jsonSource) { - return client.prepareIndex(index, type).setSource(jsonSource); - } - - // Index data - public boolean indexData(String index, String type, String jsonSource) { - if (jsonSource != null) { - IndexRequestBuilder requestBuilder = client.prepareIndex(index, type); - requestBuilder.setSource(jsonSource).execute().actionGet(); - return true; - } - return false; - } - - // Get data - public List getSearchData(String index, String type, QueryBuilder queryBuilder, int size) { - List ids = new ArrayList<>(); - SearchResponse searchResponse = client.prepareSearch(index).setTypes(type).setQuery(queryBuilder).setSize(size) - .execute().actionGet(); - SearchHits searchHits = searchResponse.getHits(); - for (SearchHit searchHit : searchHits) { - Integer id = (Integer) searchHit.getSourceAsMap().get("id"); - ids.add(id); - } - return ids; - } - - // Get data by scroll - public List getSearchDataByScrolls(String index, String type, QueryBuilder queryBuilder, int size) { - List ids = new ArrayList<>(); - SearchResponse scrollResp = client.prepareSearch(index).setTypes(type).setScroll(new TimeValue(60000)) - .setQuery(queryBuilder).setSize(size).execute().actionGet(); - while (true) { - for (SearchHit searchHit : scrollResp.getHits().getHits()) { - Integer id = (Integer) searchHit.getSourceAsMap().get("id"); - ids.add(id); - } - scrollResp = client.prepareSearchScroll(scrollResp.getScrollId()).setScroll(new TimeValue(600000)).execute() - .actionGet(); - if (scrollResp.getHits().getHits().length == 0) { - break; - } - } - - return ids; - } - - // Get results from aggregation - public Map getAggSearchResult(String index, QueryBuilder queryBuilder, - AggregationBuilder aggregationBuilder, String aggName) { - Map resultsMap = new HashMap<>(); - SearchResponse searchResponse = client.prepareSearch(index).setQuery(queryBuilder) - .addAggregation(aggregationBuilder).execute().actionGet(); - Terms terms = searchResponse.getAggregations().get(aggName); - if (terms != null) { - for (Terms.Bucket entry : terms.getBuckets()) { - resultsMap.put(entry.getKey().toString(), String.valueOf(entry.getDocCount())); - } - } - - return resultsMap; - } - - // Get results from date histogram aggregation - public Map getDateHistogramAggSearchResult(String index, QueryBuilder queryBuilder, - DateHistogramAggregationBuilder aggregationBuilder, String aggName) { - Map resultsMap = new HashMap<>(); - SearchResponse searchResponse = client.prepareSearch(index).setQuery(queryBuilder) - .addAggregation(aggregationBuilder).execute().actionGet(); - ParsedDateHistogram dateHistogram = searchResponse.getAggregations().get(aggName); - if (dateHistogram != null) { - for (Histogram.Bucket entry : dateHistogram.getBuckets()) { - resultsMap.put(entry.getKey().toString(), String.valueOf(entry.getDocCount())); - } - } - - return resultsMap; - } - - public Map getSumAggSearchOrderResult(String index, QueryBuilder queryBuilder, - AggregationBuilder aggregationBuilder, String aggName, String subAggName) { - Map resultsMap = new HashMap<>(); - SearchResponse searchResponse = client.prepareSearch(index).setQuery(queryBuilder) - .addAggregation(aggregationBuilder).execute().actionGet(); - Terms terms = searchResponse.getAggregations().get(aggName); - if (terms != null) { - for (Terms.Bucket entry : terms.getBuckets()) { - String fieldName = entry.getKey().toString(); - // long fieldCount = entry.getDocCount(); - - Sum maxagg = entry.getAggregations().get(subAggName); - double fieldValue = maxagg.getValue(); - - // System.out.println(String.format("%s %s %s", fieldName, fieldCount, fieldValue)); - resultsMap.put(fieldName, String.valueOf(fieldValue)); - } - } - - return resultsMap; - } - - public Map getMaxAggSearchOrderResult(String index, QueryBuilder queryBuilder, - AggregationBuilder aggregationBuilder, String aggName, String subAggName) { - Map resultsMap = new HashMap<>(); - SearchResponse searchResponse = client.prepareSearch(index).setQuery(queryBuilder) - .addAggregation(aggregationBuilder).execute().actionGet(); - Terms terms = searchResponse.getAggregations().get(aggName); - if (terms != null) { - for (Terms.Bucket entry : terms.getBuckets()) { - String fieldName = entry.getKey().toString(); - // long fieldCount = entry.getDocCount(); - - Max maxagg = entry.getAggregations().get(subAggName); - double fieldValue = maxagg.getValue(); - - // System.out.println(String.format("%s %s %s", fieldName, fieldCount, fieldValue)); - resultsMap.put(fieldName, String.valueOf(fieldValue)); - } - } - - return resultsMap; - } - -} diff --git a/IndexSearchService/src/main/java/com/luxx/index/executor/MainExecutor.java b/IndexSearchService/src/main/java/com/luxx/index/executor/MainExecutor.java deleted file mode 100644 index 4f3490e..0000000 --- a/IndexSearchService/src/main/java/com/luxx/index/executor/MainExecutor.java +++ /dev/null @@ -1,24 +0,0 @@ -package com.luxx.index.executor; - -import org.springframework.beans.factory.DisposableBean; -import org.springframework.beans.factory.InitializingBean; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.stereotype.Component; - -@Component -public class MainExecutor implements InitializingBean, DisposableBean { - @Autowired - @Qualifier("poiIndexExecutor") - private DataIndexExecutor dataIndexExecutor; - - @Override - public void afterPropertiesSet() throws Exception { - dataIndexExecutor.start(); - } - - @Override - public void destroy() throws Exception { - dataIndexExecutor.stop(); - } -} diff --git a/IndexSearchService/src/main/java/com/luxx/index/model/HotspotData.java b/IndexSearchService/src/main/java/com/luxx/index/model/HotspotData.java deleted file mode 100644 index d7b822a..0000000 --- a/IndexSearchService/src/main/java/com/luxx/index/model/HotspotData.java +++ /dev/null @@ -1,151 +0,0 @@ -package com.luxx.index.model; - -import java.io.Serializable; -import java.util.Date; - -public class HotspotData implements Serializable { - private static final long serialVersionUID = -2132799122365334550L; - - private Long id; - private int seqNum; - private Date collectTime; - - private String imsi; - private String imei; - private String tmsi; - - private String srcLac; - private String deviceID; - private String teleOper; - private String ownArea; - private String teleSevenNum; - private String teleBrand; - - public Long getId() { - return id; - } - - public void setId(Long newId) { - id = newId; - } - - public int getSeqNum() { - return seqNum; - } - - public void setSeqNum(int newSeqNum) { - seqNum = newSeqNum; - } - - public Date getCollectTime() { - return collectTime; - } - - public void setCollectTime(Date newCollectTime) { - collectTime = newCollectTime; - } - - public String getImsi() { - return imsi; - } - - public void setImsi(String newImsi) { - imsi = newImsi; - } - - public String getImei() { - return imei; - } - - public void setImei(String newImei) { - imei = newImei; - } - - public String getTmsi() { - return tmsi; - } - - public void setTmsi(String newTmsi) { - tmsi = newTmsi; - } - - public String getSrcLac() { - return srcLac; - } - - public void setSrcLac(String newSrcLac) { - srcLac = newSrcLac; - } - - public String getDeviceID() { - return deviceID; - } - - public void setDeviceID(String newDeviceID) { - deviceID = newDeviceID; - } - - public String getTeleOper() { - return teleOper; - } - - public void setTeleOper(String newTeleOper) { - teleOper = newTeleOper; - } - - public String getOwnArea() { - return ownArea; - } - - public void setOwnArea(String newOwnArea) { - ownArea = newOwnArea; - } - - public String getTeleSevenNum() { - return teleSevenNum; - } - - public void setTeleSevenNum(String newTeleSevenNum) { - teleSevenNum = newTeleSevenNum; - } - - public String getTeleBrand() { - return teleBrand; - } - - public void setTeleBrand(String newTeleBrand) { - teleBrand = newTeleBrand; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (null == obj || !(obj instanceof HotspotData)) { - return false; - } - HotspotData hot = (HotspotData) obj; - - if (!imei.equals(hot.getImei())) - return false; - if (!imsi.equals(hot.getImsi())) - return false; - if (collectTime.getTime() != hot.getCollectTime().getTime()) - return false; - if (!deviceID.equals(hot.getDeviceID())) - return false; - return true; - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((collectTime == null) ? 0 : collectTime.hashCode()); - result = prime * result + ((deviceID == null) ? 0 : deviceID.hashCode()); - result = prime * result + ((imei == null) ? 0 : imei.hashCode()); - result = prime * result + ((imsi == null) ? 0 : imsi.hashCode()); - return result; - } -} diff --git a/IndexSearchService/src/main/java/com/luxx/index/model/PoiData.java b/IndexSearchService/src/main/java/com/luxx/index/model/PoiData.java deleted file mode 100644 index 5ececcd..0000000 --- a/IndexSearchService/src/main/java/com/luxx/index/model/PoiData.java +++ /dev/null @@ -1,55 +0,0 @@ -package com.luxx.index.model; - -import java.io.Serializable; - -public class PoiData implements Serializable { - private static final long serialVersionUID = -3978597686509612192L; - - private long id; - private String address; - private double lat; - private double lng; - - public PoiData() { - } - - public PoiData(int id, String address, double lat, double lng) { - this.id = id; - this.address = address; - this.lat = lat; - this.lng = lng; - } - - public long getId() { - return id; - } - - public void setId(long id) { - this.id = id; - } - - public String getAddress() { - return address; - } - - public void setAddress(String address) { - this.address = address; - } - - public double getLat() { - return lat; - } - - public void setLat(double lat) { - this.lat = lat; - } - - public double getLng() { - return lng; - } - - public void setLng(double lng) { - this.lng = lng; - } - -} diff --git a/IndexSearchService/src/main/java/com/luxx/index/service/PoiIndexService.java b/IndexSearchService/src/main/java/com/luxx/index/service/PoiIndexService.java deleted file mode 100644 index 1d88a4b..0000000 --- a/IndexSearchService/src/main/java/com/luxx/index/service/PoiIndexService.java +++ /dev/null @@ -1,318 +0,0 @@ -package com.luxx.index.service; - -import com.luxx.index.model.PoiData; -import org.apache.lucene.analysis.cn.smart.SmartChineseAnalyzer; -import org.apache.lucene.document.*; -import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.index.IndexWriterConfig.OpenMode; -import org.apache.lucene.search.*; -import org.apache.lucene.search.BooleanClause.Occur; -import org.apache.lucene.spatial.SpatialStrategy; -import org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy; -import org.apache.lucene.spatial.prefix.tree.GeohashPrefixTree; -import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree; -import org.apache.lucene.spatial.query.SpatialArgs; -import org.apache.lucene.spatial.query.SpatialOperation; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.SimpleFSDirectory; -import org.apache.lucene.util.QueryBuilder; -import org.locationtech.spatial4j.context.SpatialContext; -import org.locationtech.spatial4j.distance.DistanceUtils; -import org.locationtech.spatial4j.shape.Point; -import org.locationtech.spatial4j.shape.Shape; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.locationtech.spatial4j.shape.ShapeFactory; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.stereotype.Service; -import org.wltea.analyzer.lucene.IKAnalyzer; - -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import java.io.IOException; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.List; - -@Service -public class PoiIndexService { - private static Logger log = LogManager.getLogger(PoiIndexService.class); - - private String indexPath = "D:/IndexPoiData"; - private IndexWriter indexWriter = null; - - // private IKAnalyzer ikAnalyzer = new IKAnalyzer(true); - private SmartChineseAnalyzer analyzer = new SmartChineseAnalyzer(true); - - private DirectoryReader directoryReader = null; - private Directory directory = null; - - // Spatial index and search - private SpatialContext spatialContext; - private ShapeFactory shapeFactory; - private SpatialStrategy spatialStrategy; - - // Field Name - private static final String IDFieldName = "id"; - private static final String AddressFieldName = "address"; - private static final String LatFieldName = "lat"; - private static final String LngFieldName = "lng"; - private static final String GeoFieldName = "geoField"; - - private final int maxResultCount = 100; - - @PostConstruct - public void init() { - try { - directory = new SimpleFSDirectory(Paths.get(indexPath)); - IndexWriterConfig config = new IndexWriterConfig(analyzer); - config.setOpenMode(OpenMode.CREATE_OR_APPEND); - indexWriter = new IndexWriter(directory, config); - - // Typical geo spatial context - // These can also be constructed from SpatialContextFactory - spatialContext = SpatialContext.GEO; - shapeFactory = spatialContext.getShapeFactory(); - int maxLevels = 11; // results in sub-meter precision for geohash - // This can also be constructed from SpatialPrefixTreeFactory - SpatialPrefixTree grid = new GeohashPrefixTree(spatialContext, maxLevels); - spatialStrategy = new RecursivePrefixTreeStrategy(grid, GeoFieldName); - } catch (Exception ex) { - log.error("PoiIndexService init exception: " + ex.toString()); - } - } - - private IndexSearcher getIndexSearcher() { - try { - if (directoryReader == null) { - directoryReader = DirectoryReader.open(directory); - } else { - DirectoryReader directoryReader = DirectoryReader.openIfChanged(this.directoryReader); - if (directoryReader != null) { - this.directoryReader.close(); // 关闭原reader - this.directoryReader = directoryReader; // 赋予新reader - } - } - return new IndexSearcher(directoryReader); - } catch (Exception e) { - log.error(e.toString()); - } - return null; - } - - public boolean indexPoiDataList(List dataList) { - try { - if (dataList != null && dataList.size() > 0) { - List docs = new ArrayList<>(); - for (PoiData data : dataList) { - Document doc = new Document(); - doc.add(new StoredField(IDFieldName, data.getId())); - doc.add(new StoredField(LatFieldName, data.getLat())); - doc.add(new StoredField(LngFieldName, data.getLng())); - doc.add(new TextField(AddressFieldName, data.getAddress(), Field.Store.YES)); - Point point = shapeFactory.pointXY(data.getLng(), data.getLat()); - for (Field f : spatialStrategy.createIndexableFields(point)) { - doc.add(f); - } - docs.add(doc); - } - indexWriter.addDocuments(docs); - indexWriter.commit(); - return true; - } - return false; - } catch (Exception e) { - log.error(e.toString()); - return false; - } - } - - private void printDocs(ScoreDoc[] scoreDocs, IndexSearcher indexSearcher) { - if (scoreDocs != null) { - log.info("Total count：" + scoreDocs.length); - for (ScoreDoc scoreDoc : scoreDocs) { - try { - Document hitDoc = indexSearcher.doc(scoreDoc.doc); - log.info(hitDoc.get(IDFieldName)); - log.info(hitDoc.get(LngFieldName)); - log.info(hitDoc.get(LatFieldName)); - log.info(hitDoc.get(AddressFieldName)); - } catch (IOException e) { - log.error(e.toString()); - } - } - } - } - - private List getDataFromSearchResult(ScoreDoc[] scoreDocs, IndexSearcher indexSearcher) { - List datas = new ArrayList<>(); - if (scoreDocs != null) { - log.info("Total count：" + scoreDocs.length); - for (ScoreDoc scoreDoc : scoreDocs) { - try { - Document hitDoc = indexSearcher.doc(scoreDoc.doc); - PoiData data = new PoiData(); - data.setId(Long.parseLong((hitDoc.get(IDFieldName)))); - data.setLng(Double.parseDouble(hitDoc.get(LngFieldName))); - data.setLat(Double.parseDouble(hitDoc.get(LatFieldName))); - data.setAddress(hitDoc.get(AddressFieldName)); - datas.add(data); - } catch (IOException e) { - log.error(e.toString()); - } - } - } - - return datas; - } - - private int doQuery(Query query, IndexSearcher indexSearcher) { - TopDocs hits = null; - try { - hits = indexSearcher.search(query, maxResultCount); - } catch (IOException e) { - log.error(e.toString()); - } - if (hits != null) { - ScoreDoc[] scoreDocs = hits.scoreDocs; - printDocs(scoreDocs, indexSearcher); - return scoreDocs.length; - } - return 0; - } - - public int searchPoiDataByAddress(String address) { - IndexSearcher indexSearcher = getIndexSearcher(); - if (indexSearcher != null) { - QueryBuilder builder = new QueryBuilder(analyzer); - Query query = builder.createPhraseQuery(AddressFieldName, address); - return doQuery(query, indexSearcher); - } - return 0; - } - - public List searchPoiInRectangle(double minLng, double minLat, double maxLng, double maxLat) { - List results = new ArrayList<>(); - IndexSearcher indexSearcher = getIndexSearcher(); - if (indexSearcher != null) { - Point lowerLeftPoint = shapeFactory.pointXY(minLng, minLat); - Point upperRightPoint = shapeFactory.pointXY(maxLng, maxLat); - Shape rect = shapeFactory.rect(lowerLeftPoint, upperRightPoint); - - SpatialArgs args = new SpatialArgs(SpatialOperation.Intersects, rect); - Query query = spatialStrategy.makeQuery(args); - TopDocs docs = null; - try { - docs = indexSearcher.search(query, maxResultCount); - } catch (IOException e) { - log.error(e.toString()); - } - - if (docs != null) { - ScoreDoc[] scoreDocs = docs.scoreDocs; - //printDocs(scoreDocs, indexSearcher); - results = getDataFromSearchResult(scoreDocs, indexSearcher); - } - } - - return results; - } - - public List searchPoiInCircle(double lng, double lat, double radius) { - List results = new ArrayList<>(); - IndexSearcher indexSearcher = getIndexSearcher(); - if (indexSearcher != null) { - Point pt = shapeFactory.pointXY(lng, lat); - Shape circle = shapeFactory.circle(pt, DistanceUtils.dist2Degrees( - radius, DistanceUtils.EARTH_MEAN_RADIUS_KM)); - SpatialArgs args = new SpatialArgs(SpatialOperation.Intersects, circle); - Query query = spatialStrategy.makeQuery(args); - // the distance (in km) - DoubleValuesSource valueSource = spatialStrategy.makeDistanceValueSource(pt, DistanceUtils.DEG_TO_KM); - Sort distSort = null; - TopDocs docs = null; - try { - // false = asc dist - distSort = new Sort(valueSource.getSortField(false)).rewrite(indexSearcher); - docs = indexSearcher.search(query, maxResultCount, distSort); - } catch (IOException e) { - log.error(e.toString()); - } - - if (docs != null) { - ScoreDoc[] scoreDocs = docs.scoreDocs; - //printDocs(scoreDocs, indexSearcher); - results = getDataFromSearchResult(scoreDocs, indexSearcher); - } - } - - return results; - } - - public List searchPoiInCircleAndAddress(double lng, double lat, double radius, String address) { - List results = new ArrayList<>(); - IndexSearcher indexSearcher = getIndexSearcher(); - if (indexSearcher != null) { - Point pt = shapeFactory.pointXY(lng, lat); - Shape circle = shapeFactory.circle(pt, DistanceUtils.dist2Degrees( - radius, DistanceUtils.EARTH_MEAN_RADIUS_KM)); - SpatialArgs args = new SpatialArgs(SpatialOperation.Intersects, circle); - Query geoQuery = spatialStrategy.makeQuery(args); - - QueryBuilder builder = new QueryBuilder(analyzer); - Query phraseQuery = builder.createPhraseQuery(AddressFieldName, address); - BooleanQuery.Builder boolBuilder = new BooleanQuery.Builder(); - boolBuilder.add(phraseQuery, Occur.SHOULD); - boolBuilder.add(geoQuery, Occur.MUST); - - Query query = boolBuilder.build(); - TopDocs docs = null; - try { - docs = indexSearcher.search(query, maxResultCount); - } catch (IOException e) { - log.error(e.toString()); - } - - if (docs != null) { - ScoreDoc[] scoreDocs = docs.scoreDocs; - //printDocs(scoreDocs, indexSearcher); - results = getDataFromSearchResult(scoreDocs, indexSearcher); - } - } - return results; - } - - @PreDestroy - public void close() { - if (indexWriter != null) { - try { - indexWriter.close(); - } catch (IOException e) { - log.error(e.toString()); - } - } - - if (directoryReader != null) { - try { - directoryReader.close(); - } catch (IOException e) { - log.error(e.toString()); - } - } - } - - public void clear() { - if (indexWriter != null) { - try { - log.info("Delete all exist data"); - indexWriter.deleteAll(); - } catch (IOException e) { - log.error(e.toString()); - } - } - } - -} diff --git a/IndexSearchService/src/main/resources/application.properties b/IndexSearchService/src/main/resources/application.properties deleted file mode 100644 index 7ec9d63..0000000 --- a/IndexSearchService/src/main/resources/application.properties +++ /dev/null @@ -1,9 +0,0 @@ -# ES -#index.es.cluster.name=es-poi -#index.es.cluster.address=127.0.0.1:9300 -# POI index DB -index.db.url=jdbc:mysql://127.0.0.1:3306/data?useLegacyDatetimeCode=false&serverTimezone=Asia/Shanghai -index.db.username=data -index.db.password=123 -index.db.table=station -index.db.type=mysql \ No newline at end of file diff --git a/IndexSearchService/src/main/resources/log4j2.xml b/IndexSearchService/src/main/resources/log4j2.xml deleted file mode 100644 index ff90882..0000000 --- a/IndexSearchService/src/main/resources/log4j2.xml +++ /dev/null @@ -1,28 +0,0 @@ - - - - ${sys:user.dir}/logs/ - data-transfer - - - - - %d %p %c [%t] %m%n - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/IndexSearchService/src/test/java/com/luxx/index/test/MysqlIndexServiceTest.java b/IndexSearchService/src/test/java/com/luxx/index/test/MysqlIndexServiceTest.java deleted file mode 100644 index 4dc8582..0000000 --- a/IndexSearchService/src/test/java/com/luxx/index/test/MysqlIndexServiceTest.java +++ /dev/null @@ -1,130 +0,0 @@ -package com.luxx.index.test; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import com.luxx.index.service.MysqlIndexService; - -public class MysqlIndexServiceTest { - - public void TestGetDataBetweenTime() { - MysqlIndexService indexService = new MysqlIndexService(); - // CollectTime:[2014-12-06T00:00:00Z TO 2014-12-10T21:31:55Z] - // "2014-12-16T07:37:21.000Z" - String startTime = "2014-12-06 00:00:00"; - // String endTime = "2014-12-11 21:31:55"; - String endTime = "2014-12-13 00:00:00"; - long s = System.currentTimeMillis(); - List results = indexService.getSearchResultBetweenTime(startTime, endTime); - if (results != null) { - System.out.println("Size: " + results.size()); - } - long e = System.currentTimeMillis(); - System.out.println("Time: " + (e - s)); - - indexService.close(); - } - - public void TestDateFacet() { - MysqlIndexService indexService = new MysqlIndexService(); - long s = System.currentTimeMillis(); - String startTime = "2015-01-01 00:00:00"; - String endTime = "2015-12-30 21:31:55"; - List deviceList = new ArrayList(); - deviceList.add("1011"); - deviceList.add("1001"); - deviceList.add("1007"); - Map response = new HashMap<>(); - response = indexService.getDeviceDateFacetDistributeInfo(startTime, endTime, null, "Month"); - if (response != null) { - for (String key : response.keySet()) { - System.out.println(key + " : " + response.get(key)); - } - } - long e = System.currentTimeMillis(); - System.out.println("Time: " + (e - s)); - indexService.close(); - } - - public void TestDeviceFacet() { - MysqlIndexService indexService = new MysqlIndexService(); - long s = System.currentTimeMillis(); - String startTime = "2014-12-06 00:00:00"; - String endTime = "2014-12-10 21:31:55"; - List deviceList = new ArrayList(); - deviceList.add("1011"); - deviceList.add("1001"); - deviceList.add("1007"); - Map response = new HashMap<>(); - response = indexService.getDeviceDistributedInfo(startTime, endTime, null); - if (response != null) { - // System.out.println("Search Results: "+response.size()); - for (String key : response.keySet()) { - System.out.println(key + " : " + response.get(key)); - } - } - long e = System.currentTimeMillis(); - System.out.println("Time: " + (e - s)); - indexService.close(); - } - - public void TestOwnAreaFacet() { - MysqlIndexService indexService = new MysqlIndexService(); - long s = System.currentTimeMillis(); - String startTime = "2014-12-06 00:00:00"; - String endTime = "2014-12-10 21:31:55"; - List deviceList = new ArrayList(); - deviceList.add("1011"); - deviceList.add("1001"); - deviceList.add("1007"); - Map response = new HashMap<>(); - // response = indexService.getOwnAreaDistributeInfo(startTime, endTime, null,"上海",null); - // response = indexService.getOwnAreaDistributeInfo(startTime, endTime, null,"河北","唐山"); - response = indexService.getOwnAreaDistributeInfo(startTime, endTime, null, null, null); - if (response != null) { - // System.out.println("Search Results: "+response.size()); - int sum = 0; - int count = 0; - for (String key : response.keySet()) { - System.out.println(key + " : " + response.get(key)); - sum += Integer.parseInt(response.get(key)); - ++count; - } - System.out.println("Count: " + count); - System.out.println("Sum: " + sum); - } - - long e = System.currentTimeMillis(); - System.out.println("Time: " + (e - s)); - indexService.close(); - } - - public void TestTelOperFieldFacet() { - MysqlIndexService indexService = new MysqlIndexService(); - long s = System.currentTimeMillis(); - String startTime = "2014-12-06 00:00:00"; - String endTime = "2014-12-10 21:31:55"; - List deviceList = new ArrayList(); - deviceList.add("1011"); - deviceList.add("1001"); - deviceList.add("1007"); - Map response = new HashMap<>(); - response = indexService.getTeleOperDistributeInfo(startTime, endTime, null); - if (response != null) { - // System.out.println("Search Results: "+response.size()); - int sum = 0; - for (String key : response.keySet()) { - System.out.println(key + " : " + response.get(key)); - sum += Integer.parseInt(response.get(key)); - } - System.out.println("Sum: " + sum); - } - - long e = System.currentTimeMillis(); - System.out.println("Time: " + (e - s)); - indexService.close(); - } - -} diff --git a/IndexSearchService/src/test/java/com/luxx/index/test/RedshiftIndexServiceTest.java b/IndexSearchService/src/test/java/com/luxx/index/test/RedshiftIndexServiceTest.java deleted file mode 100644 index 7acad67..0000000 --- a/IndexSearchService/src/test/java/com/luxx/index/test/RedshiftIndexServiceTest.java +++ /dev/null @@ -1,40 +0,0 @@ -package com.luxx.index.test; - -import com.luxx.index.service.RedshiftIndexService; - -import java.util.Map; - -public class RedshiftIndexServiceTest { - - public static void test1(RedshiftIndexService indexService) { - long s = System.currentTimeMillis(); - Map resultsMap = indexService.getTopNEndpointUsage("2017-1-1 00:00:00", "2017-3-1 00:00:00", - 7564, 5); - System.out.println("Endpoint id : sum of usage"); - for (String key : resultsMap.keySet()) { - System.out.println(key + " : " + resultsMap.get(key)); - } - long e = System.currentTimeMillis(); - System.out.println("Time: " + (e - s)); - } - - public static void test2(RedshiftIndexService indexService) { - long s = System.currentTimeMillis(); - Map resultsMap = indexService.getEndpointMaxbps("2017-1-1 00:00:00", "2017-3-1 00:00:00", 7564); - System.out.println("Endpoint id : max bps"); - for (String key : resultsMap.keySet()) { - System.out.println(key + " : " + resultsMap.get(key)); - } - long e = System.currentTimeMillis(); - System.out.println("Time: " + (e - s)); - } - - public static void main(String[] args) { - RedshiftIndexService indexService = new RedshiftIndexService(); - - test1(indexService); - test2(indexService); - - } - -} diff --git a/LogCollector/Dockerfile b/LogCollector/Dockerfile index 87dda4e..4dfc094 100644 --- a/LogCollector/Dockerfile +++ b/LogCollector/Dockerfile @@ -1,4 +1,4 @@ -FROM soft.luxx.cn/library/openjdk-18-el7:latest +FROM adoptopenjdk/openjdk8:alpine WORKDIR /service COPY target/log-collector-1.0.jar /service/log-collector.jar COPY entrypoint.sh /service/entrypoint.sh diff --git a/LogCollector/docker-compose.yml b/LogCollector/docker-compose.yml index 834b90e..7dd6224 100644 --- a/LogCollector/docker-compose.yml +++ b/LogCollector/docker-compose.yml @@ -1,10 +1,12 @@ -version: '1' +version: '3' services: log-collector: + container_name: log-collector image: log-collector build: ./ environment: - JAVA_OPTS=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8899 + - _JAVA_OPTIONS=-Xms1024m -Xmx2048m - LOG_LEVEL=info - ENV=dev - ES_ADDRESS=10.10.10.10:9200 diff --git a/LogCollector/pom.xml b/LogCollector/pom.xml index fdef715..2cd3ab3 100644 --- a/LogCollector/pom.xml +++ b/LogCollector/pom.xml @@ -5,7 +5,7 @@ org.springframework.boot spring-boot-starter-parent - 2.2.7.RELEASE + 2.5.12 @@ -17,13 +17,12 @@ true 1.8 - 2.9.2 - 2.2.7.RELEASE + 2.5.12 1.18.16 - 7.6.2 - 30.1-jre - 2.12.4 - 2.8.6 + 7.17.14 + 2.14.0 + 32.0.0-jre + 2.8.9 4.13.1 @@ -74,13 +73,6 @@ ${es.version} - - - com.fasterxml.jackson.core - jackson-databind - ${jackson.version} - - org.projectlombok lombok @@ -100,6 +92,13 @@ ${guava.version} + + + com.fasterxml.jackson.core + jackson-databind + ${jackson.version} + + com.google.code.gson gson diff --git a/LogCollector/src/main/resources/log4j2-dev.xml b/LogCollector/src/main/resources/log4j2-dev.xml index e4fa422..071d947 100644 --- a/LogCollector/src/main/resources/log4j2-dev.xml +++ b/LogCollector/src/main/resources/log4j2-dev.xml @@ -8,16 +8,20 @@ - - + - + + + + + + diff --git a/LogCollector/src/main/resources/log4j2-prod.xml b/LogCollector/src/main/resources/log4j2-prod.xml index e4fa422..be0d90e 100644 --- a/LogCollector/src/main/resources/log4j2-prod.xml +++ b/LogCollector/src/main/resources/log4j2-prod.xml @@ -15,9 +15,14 @@ - + - + + + + + + diff --git a/MapHttpService/pom.xml b/MapHttpService/pom.xml index c6f0234..ded10d5 100644 --- a/MapHttpService/pom.xml +++ b/MapHttpService/pom.xml @@ -18,9 +18,9 @@ UTF-8 1.7.31 4.1.42.Final - 30.1-jre + 32.0.0-jre 3.4.5 - 8.0.16 + 8.0.28 @@ -62,7 +62,7 @@ org.xerial sqlite-jdbc - 3.7.2 + 3.41.2.2 diff --git a/NettyMqService/pom.xml b/NettyMqService/pom.xml index 09c472e..7180ccc 100644 --- a/NettyMqService/pom.xml +++ b/NettyMqService/pom.xml @@ -18,7 +18,7 @@ UTF-8 1.7.31 4.1.42.Final - 30.1-jre + 32.0.0-jre diff --git a/README.md b/README.md index 2bbddd2..37bee54 100644 --- a/README.md +++ b/README.md @@ -6,30 +6,29 @@ How to implement a message queue service with Netty and RabbitMQ. ##### 中文详情(Chinese Details): * [Message Queue Service based on Netty and RabbitMQ](http://www.cnblogs.com/luxiaoxun/p/4257105.html) -## 2. IndexSearchService +## 2. MapHttpService +A simple Http Map Service providing tile image for Map. + +## 3. WebGisDemo +A simple web GIS page based on [leaflet](https://github.com/Leaflet/Leaflet). +##### 中文详情([Chinese Details](http://www.cnblogs.com/luxiaoxun/p/5022333.html)) 1. How to use Elasticsearch. 2. How to use Solr or Lucene to index and query data. -3. How to use Lucene to index and query POI(GEO data). -##### 中文详情(Chinese Details): +3. How to use Elasticsearch to index and query POI(GEO data). + * [Lucene index and query POI data](http://www.cnblogs.com/luxiaoxun/p/5020247.html) * [Solr index and query MYSQL data](http://www.cnblogs.com/luxiaoxun/p/4442770.html) * [Solr index and query GEO data](http://www.cnblogs.com/luxiaoxun/p/4477591.html) * [Elasticsearch index and query data](http://www.cnblogs.com/luxiaoxun/p/4869509.html) * [SQL to Elasticsearch Java Code](http://www.cnblogs.com/luxiaoxun/p/6826211.html) -## 3. MapHttpService -A simple Http Map Service providing tile image for Map. - -## 4. WebGisDemo -A simple web GIS page based on [leaflet](https://github.com/Leaflet/Leaflet). -##### 中文详情([Chinese Details](http://www.cnblogs.com/luxiaoxun/p/5022333.html)) How to use WebGisDemo with MapHttpService: 1. Download map tile image with [MapDownloader](https://github.com/luxiaoxun/MapDownloader) 2. Start MapHttpService with right data source, an example: >port=8899 >database.type=0 # Use SQLite >database.sqlite.path=E:\\GIS\\MapDownloader\\MapCache\\TileDBv5\\en\\Data.gmdb -3. Start WebGISDemo: http://localhost:9090/map/map +3. Start WebGISDemo: http://localhost:9090/map ![map](https://github.com/luxiaoxun/Code4Java/blob/master/WebGisDemo/picture/map-demo.png) diff --git a/WebGisDemo/db_table/db_table.sql b/WebGisDemo/db_table/db_table.sql new file mode 100644 index 0000000..afdde2c --- /dev/null +++ b/WebGisDemo/db_table/db_table.sql @@ -0,0 +1,31 @@ +CREATE DATABASE `data_db`; + +USE `data_db`; + +DROP TABLE IF EXISTS `station`; +CREATE TABLE `station` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `country_code` varchar(16) DEFAULT '', + `provider` varchar(32) DEFAULT '', + `lac` varchar(32) DEFAULT '', + `cell` varchar(32) DEFAULT '', + `latitude` DOUBLE(20,8) DEFAULT NULL, + `longitude` DOUBLE(20,8) DEFAULT NULL, + `address` varchar(512) DEFAULT '', + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='station table' + +INSERT INTO `station` (`id`, `country_code`, `provider`, `lac`, `cell`, `latitude`, `longitude`, `address`) +VALUES ('1', '406', '0', '111', '222', '32.12800000', '118.77420000', '南京测试点1'); + +INSERT INTO `station` (`id`, `country_code`, `provider`, `lac`, `cell`, `latitude`, `longitude`, `address`) +VALUES ('2', '406', '1', '123', '234', '32.22700000', '118.86420000', '南京测试点2'); + +INSERT INTO `station` (`id`, `country_code`, `provider`, `lac`, `cell`, `latitude`, `longitude`, `address`) +VALUES ('3', '406', '0', '222', '333', '30.32600000', '118.78420000', '南京测试点3'); + +INSERT INTO `station` (`id`, `country_code`, `provider`, `lac`, `cell`, `latitude`, `longitude`, `address`) +VALUES ('4', '406', '1', '123', '234', '31.22700000', '118.56420000', '南京测试点4'); + +INSERT INTO `station` (`id`, `country_code`, `provider`, `lac`, `cell`, `latitude`, `longitude`, `address`) +VALUES ('5', '406', '0', '222', '333', '32.32600000', '118.68420000', '南京测试点5'); diff --git a/WebGisDemo/pom.xml b/WebGisDemo/pom.xml index 3de90bb..d06d8c6 100644 --- a/WebGisDemo/pom.xml +++ b/WebGisDemo/pom.xml @@ -16,24 +16,27 @@ UTF-8 UTF-8 UTF-8 - 2.2.7.RELEASE - 2.1.4 - 1.2.13 - 2.12.1 - 2.9.2 + 2.5.12 + 2.2.2 + 1.4.2 + 7.17.14 + 2.18.0 + 2.14.0 + 3.0.0 1.18.16 - 30.0-jre + 32.0.0-jre 3.4.5 - 8.0.16 + 8.0.28 - - - com.luxx - IndexSearchService - 1.0.0 - + + + redshift + https://redshift-maven-repository.s3-website-us-east-1.amazonaws.com/release + + + org.springframework.boot @@ -51,11 +54,27 @@ spring-boot-starter-web ${spring-boot.version} + + org.springframework.boot + spring-boot-starter-validation + ${spring-boot.version} + org.springframework.boot spring-boot-starter-log4j2 ${spring-boot.version} + + + org.apache.logging.log4j + log4j-api + ${log4j2.version} + + + org.apache.logging.log4j + log4j-core + ${log4j2.version} + org.springframework.boot @@ -63,29 +82,17 @@ ${spring-boot.version} - - - - - - - - - org.springframework.boot - spring-boot-starter-jdbc - ${spring-boot.version} - org.mybatis.spring.boot mybatis-spring-boot-starter - ${mybatis.spring.boot.version} + ${mybatis.spring-boot.version} com.github.pagehelper pagehelper-spring-boot-starter - ${pagehelper.spring.boot.version} + ${pagehelper.spring-boot.version} @@ -101,6 +108,30 @@ ${mysql.version} + + + com.fasterxml.jackson.core + jackson-databind + ${jackson.version} + + + + + org.elasticsearch + elasticsearch + ${es.version} + + + org.elasticsearch.client + elasticsearch-rest-client + ${es.version} + + + org.elasticsearch.client + elasticsearch-rest-high-level-client + ${es.version} + + com.google.guava @@ -111,12 +142,7 @@ io.springfox - springfox-swagger2 - ${swagger.version} - - - io.springfox - springfox-swagger-ui + springfox-boot-starter ${swagger.version} @@ -128,19 +154,6 @@ - - - - - org.springframework.boot - spring-boot-dependencies - ${spring-boot.version} - pom - import - - - - @@ -157,6 +170,9 @@ org.springframework.boot spring-boot-maven-plugin ${spring-boot.version} + + true + diff --git a/WebGisDemo/src/main/java/com/luxx/gis/WebGisApp.java b/WebGisDemo/src/main/java/com/luxx/gis/WebGisApp.java new file mode 100644 index 0000000..5453082 --- /dev/null +++ b/WebGisDemo/src/main/java/com/luxx/gis/WebGisApp.java @@ -0,0 +1,33 @@ +package com.luxx.gis; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.SpringBootConfiguration; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.web.servlet.config.annotation.EnableWebMvc; +import org.springframework.web.servlet.config.annotation.ResourceHandlerRegistry; +import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; + +@SpringBootApplication +@SpringBootConfiguration +@EnableWebMvc +@ComponentScan(basePackages = "com.luxx") +public class WebGisApp implements WebMvcConfigurer { + private static final String[] CLASSPATH_RESOURCE_LOCATIONS = { + "classpath:/META-INF/resources/", "classpath:/resources/", + "classpath:/static/", "classpath:/public/"}; + + public static void main(String[] args) { + SpringApplication.run(WebGisApp.class, args); + } + + @Override + public void addResourceHandlers(ResourceHandlerRegistry registry) { + registry.addResourceHandler("swagger-ui.html") + .addResourceLocations("classpath:/META-INF/resources/"); + registry.addResourceHandler("/webjars/**") + .addResourceLocations("classpath:/META-INF/resources/webjars/"); + registry.addResourceHandler("/static/**") + .addResourceLocations(CLASSPATH_RESOURCE_LOCATIONS); + } +} diff --git a/WebGisDemo/src/main/java/com/luxx/gis/client/ElasticSearchClient.java b/WebGisDemo/src/main/java/com/luxx/gis/client/ElasticSearchClient.java new file mode 100644 index 0000000..4500aa0 --- /dev/null +++ b/WebGisDemo/src/main/java/com/luxx/gis/client/ElasticSearchClient.java @@ -0,0 +1,365 @@ +package com.luxx.gis.client; + +import java.io.IOException; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.http.HttpHost; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.impl.nio.client.HttpAsyncClientBuilder; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; +import org.elasticsearch.action.bulk.*; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.search.*; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.client.*; +import org.elasticsearch.client.indices.CreateIndexRequest; +import org.elasticsearch.client.indices.CreateIndexResponse; +import org.elasticsearch.client.indices.PutMappingRequest; +import org.elasticsearch.common.unit.ByteSizeUnit; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.search.Scroll; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.aggregations.AggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; +import org.elasticsearch.search.aggregations.bucket.histogram.ParsedDateHistogram; +import org.elasticsearch.search.aggregations.bucket.terms.Terms; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.search.aggregations.metrics.Max; +import org.elasticsearch.search.aggregations.metrics.Sum; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Component; +import org.springframework.util.StringUtils; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; + +@Component +@Lazy +public class ElasticSearchClient { + private static Logger log = LogManager.getLogger(ElasticSearchClient.class); + + @Value("${es.address}") + private String esAddress; + + @Value("${es.username:}") + private String username; + + @Value("${es.password:}") + private String password; + + // ES Client + private RestHighLevelClient client; + private BulkProcessor bulkProcessor; + private int bulkSize = 5; + private int bulkActions = 1000; + private int flushInterval = 3; + private int concurrentRequests = 5; + + @PostConstruct + public void init() throws UnknownHostException { + log.info("es.address: " + esAddress); + String[] hostPort = esAddress.split(":"); + if (!StringUtils.isEmpty(username) && !StringUtils.isEmpty(password)) { + client = new RestHighLevelClient(RestClient.builder(new HttpHost(hostPort[0], Integer.parseInt(hostPort[1]))) + .setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() { + @Override + public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) { + CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials(AuthScope.ANY, + new UsernamePasswordCredentials(username, password)); + return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider); + } + })); + } else { + client = new RestHighLevelClient(RestClient.builder(new HttpHost(hostPort[0], Integer.parseInt(hostPort[1])))); + } + + BulkProcessor.Listener listener = new BulkProcessor.Listener() { + @Override + public void beforeBulk(long executionId, BulkRequest request) { + log.debug("ES before bulk, number of actions: {}", request.numberOfActions()); + } + + @Override + public void afterBulk(long executionId, BulkRequest request, BulkResponse responses) { + if (responses.hasFailures()) { + try { + StringBuilder sb = new StringBuilder("Failure in bulk execution: "); + int pos = responses.getItems().length - 1; + BulkItemResponse response = responses.getItems()[pos]; + while (!response.isFailed() && pos >= 0) { + response = responses.getItems()[pos]; + --pos; + } + sb.append("\n[").append(pos) + .append("]: index [").append(response.getIndex()) + .append("], type [").append(response.getType()) + .append("], id [").append(response.getId()) + .append("], message [").append(response.getFailureMessage()) + .append("]"); + log.warn(sb.toString()); + } catch (Exception e) { + log.warn("Print es bulk error failed: {}", e.toString()); + if (log.isDebugEnabled()) + log.debug("ES bulk error: {}", responses.buildFailureMessage()); + } + } + } + + @Override + public void afterBulk(long executionId, BulkRequest request, Throwable failure) { + log.error("ES bulk got exception: {}", failure.toString()); + } + }; + + bulkProcessor = BulkProcessor.builder( + (request, bulkListener) -> client.bulkAsync(request, RequestOptions.DEFAULT, bulkListener), + listener, "ES-bulk-processor") + .setBulkActions(bulkActions) + .setBulkSize(new ByteSizeValue(bulkSize, ByteSizeUnit.MB)) + .setFlushInterval(TimeValue.timeValueSeconds(flushInterval)) + .setConcurrentRequests(concurrentRequests) + .build(); + } + + @PreDestroy + public void close() { + if (client != null) { + try { + client.close(); + } catch (IOException e) { + log.error(e.toString()); + } + } + } + + public RestHighLevelClient getClient() { + return this.client; + } + + // Create Index + public boolean createIndex(String indexName) { + if (StringUtils.isEmpty(indexName)) { + return false; + } + try { + org.elasticsearch.client.indices.CreateIndexRequest request = new CreateIndexRequest(indexName); + CreateIndexResponse createIndexResponse = client.indices().create(request, RequestOptions.DEFAULT); + return createIndexResponse.isAcknowledged(); + } catch (Exception e) { + log.error("Create index error: " + e.toString()); + return false; + } + } + + // Delete Index + public boolean deleteIndex(String indexName) { + if (StringUtils.isEmpty(indexName)) { + return false; + } + try { + DeleteIndexRequest request = new DeleteIndexRequest(indexName); + AcknowledgedResponse deleteIndexResponse = client.indices() + .delete(request, RequestOptions.DEFAULT); + return deleteIndexResponse.isAcknowledged(); + } catch (Exception ex) { + log.error("Delete index {} error: {}", indexName, ex); + return false; + } + } + + // Define index mapping + public void createIndexMapping(String index, XContentBuilder contentBuilder) { + PutMappingRequest request = new PutMappingRequest(index); + request.source(contentBuilder); + try { + client.indices().putMapping(request, RequestOptions.DEFAULT); + } catch (IOException e) { + log.error("Put mapping error: " + e.toString()); + } + } + + public BulkProcessor getBulkRequest() { + return this.bulkProcessor; + } + + public IndexRequest getIndexRequest(String index, String jsonSource) { + return new IndexRequest(index).source(jsonSource); + } + + // Get data + public List getSearchData(String index, QueryBuilder queryBuilder, int size) { + List ids = new ArrayList<>(); + SearchRequest searchRequest = new SearchRequest(index); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + sourceBuilder.query(queryBuilder); + sourceBuilder.size(size); + searchRequest.source(sourceBuilder); + try { + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + SearchHits searchHits = searchResponse.getHits(); + for (SearchHit searchHit : searchHits) { + Integer id = (Integer) searchHit.getSourceAsMap().get("id"); + ids.add(id); + } + } catch (IOException e) { + log.error(e); + } + return ids; + } + + // Get data by scroll + public List getSearchDataByScrolls(String index, QueryBuilder queryBuilder) { + List ids = new ArrayList<>(); + final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(1L)); + SearchRequest searchRequest = new SearchRequest(index); + searchRequest.scroll(TimeValue.timeValueMinutes(1L)); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + sourceBuilder.query(queryBuilder); + searchRequest.source(sourceBuilder); + try { + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + String scrollId = searchResponse.getScrollId(); + SearchHit[] searchHits = searchResponse.getHits().getHits(); + while (searchHits != null && searchHits.length > 0) { + SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId); + scrollRequest.scroll(scroll); + searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT); + scrollId = searchResponse.getScrollId(); + searchHits = searchResponse.getHits().getHits(); + for (SearchHit searchHit : searchHits) { + Integer id = (Integer) searchHit.getSourceAsMap().get("id"); + ids.add(id); + } + } + ClearScrollRequest clearScrollRequest = new ClearScrollRequest(); + clearScrollRequest.addScrollId(scrollId); + ClearScrollResponse clearScrollResponse = client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT); + boolean succeeded = clearScrollResponse.isSucceeded(); + } catch (Exception ex) { + log.error(ex); + } + return ids; + } + + // Get results from aggregation + public Map getAggSearchResult(String index, QueryBuilder queryBuilder, + AggregationBuilder aggregationBuilder, String aggName) { + Map resultsMap = new HashMap<>(); + SearchRequest searchRequest = new SearchRequest(index); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + sourceBuilder.query(queryBuilder); + sourceBuilder.aggregation(aggregationBuilder); + searchRequest.source(sourceBuilder); + + try { + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + Terms terms = searchResponse.getAggregations().get(aggName); + if (terms != null) { + for (Terms.Bucket entry : terms.getBuckets()) { + resultsMap.put(entry.getKey().toString(), String.valueOf(entry.getDocCount())); + } + } + } catch (Exception ex) { + log.error(ex); + } + + return resultsMap; + } + + // Get results from date histogram aggregation + public Map getDateHistogramAggSearchResult(String index, QueryBuilder queryBuilder, + DateHistogramAggregationBuilder aggregationBuilder, String aggName) { + Map resultsMap = new HashMap<>(); + SearchRequest searchRequest = new SearchRequest(index); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + sourceBuilder.query(queryBuilder); + sourceBuilder.aggregation(aggregationBuilder); + searchRequest.source(sourceBuilder); + try { + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + ParsedDateHistogram dateHistogram = searchResponse.getAggregations().get(aggName); + if (dateHistogram != null) { + for (Histogram.Bucket entry : dateHistogram.getBuckets()) { + resultsMap.put(entry.getKey().toString(), String.valueOf(entry.getDocCount())); + } + } + } catch (Exception ex) { + log.error(ex); + } + + return resultsMap; + } + + public Map getSumAggSearchOrderResult(String index, QueryBuilder queryBuilder, + AggregationBuilder aggregationBuilder, String aggName, String subAggName) { + Map resultsMap = new HashMap<>(); + SearchRequest searchRequest = new SearchRequest(index); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + sourceBuilder.query(queryBuilder); + sourceBuilder.aggregation(aggregationBuilder); + searchRequest.source(sourceBuilder); + try { + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + Terms terms = searchResponse.getAggregations().get(aggName); + if (terms != null) { + for (Terms.Bucket entry : terms.getBuckets()) { + String fieldName = entry.getKey().toString(); + // long fieldCount = entry.getDocCount(); + Sum maxagg = entry.getAggregations().get(subAggName); + double fieldValue = maxagg.getValue(); + // System.out.println(String.format("%s %s %s", fieldName, fieldCount, fieldValue)); + resultsMap.put(fieldName, String.valueOf(fieldValue)); + } + } + } catch (Exception ex) { + log.error(ex); + } + return resultsMap; + } + + public Map getMaxAggSearchOrderResult(String index, QueryBuilder queryBuilder, + AggregationBuilder aggregationBuilder, String aggName, String subAggName) { + Map resultsMap = new HashMap<>(); + SearchRequest searchRequest = new SearchRequest(index); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + sourceBuilder.query(queryBuilder); + sourceBuilder.aggregation(aggregationBuilder); + searchRequest.source(sourceBuilder); + try { + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + Terms terms = searchResponse.getAggregations().get(aggName); + if (terms != null) { + for (Terms.Bucket entry : terms.getBuckets()) { + String fieldName = entry.getKey().toString(); + // long fieldCount = entry.getDocCount(); + Max maxagg = entry.getAggregations().get(subAggName); + double fieldValue = maxagg.getValue(); + // System.out.println(String.format("%s %s %s", fieldName, fieldCount, fieldValue)); + resultsMap.put(fieldName, String.valueOf(fieldValue)); + } + } + } catch (Exception ex) { + log.error(ex); + } + return resultsMap; + } + +} diff --git a/IndexSearchService/src/main/java/com/luxx/index/config/IndexConfig.java b/WebGisDemo/src/main/java/com/luxx/gis/config/IndexConfig.java similarity index 97% rename from IndexSearchService/src/main/java/com/luxx/index/config/IndexConfig.java rename to WebGisDemo/src/main/java/com/luxx/gis/config/IndexConfig.java index 577f501..86fddf0 100644 --- a/IndexSearchService/src/main/java/com/luxx/index/config/IndexConfig.java +++ b/WebGisDemo/src/main/java/com/luxx/gis/config/IndexConfig.java @@ -1,4 +1,4 @@ -package com.luxx.index.config; +package com.luxx.gis.config; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; diff --git a/WebGisDemo/src/main/java/com/luxx/web/config/SwaggerConfig.java b/WebGisDemo/src/main/java/com/luxx/gis/config/SwaggerConfig.java similarity index 89% rename from WebGisDemo/src/main/java/com/luxx/web/config/SwaggerConfig.java rename to WebGisDemo/src/main/java/com/luxx/gis/config/SwaggerConfig.java index ef4ebfb..fd35bf9 100644 --- a/WebGisDemo/src/main/java/com/luxx/web/config/SwaggerConfig.java +++ b/WebGisDemo/src/main/java/com/luxx/gis/config/SwaggerConfig.java @@ -1,4 +1,4 @@ -package com.luxx.web.config; +package com.luxx.gis.config; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -8,17 +8,15 @@ import springfox.documentation.service.ApiInfo; import springfox.documentation.spi.DocumentationType; import springfox.documentation.spring.web.plugins.Docket; -import springfox.documentation.swagger2.annotations.EnableSwagger2; @Configuration -@EnableSwagger2 public class SwaggerConfig { @Bean public Docket createRestApi() { return new Docket(DocumentationType.SWAGGER_2) .apiInfo(apiInfo()) .select() - .apis(RequestHandlerSelectors.basePackage("com.luxx.web")) + .apis(RequestHandlerSelectors.basePackage("com.luxx.gis")) .paths(PathSelectors.any()) .build(); } diff --git a/WebGisDemo/src/main/java/com/luxx/gis/controller/DataIndexController.java b/WebGisDemo/src/main/java/com/luxx/gis/controller/DataIndexController.java new file mode 100644 index 0000000..998ad02 --- /dev/null +++ b/WebGisDemo/src/main/java/com/luxx/gis/controller/DataIndexController.java @@ -0,0 +1,28 @@ +package com.luxx.gis.controller; + +import com.luxx.gis.executor.IndexExecutor; +import com.luxx.gis.model.ResultData; +import io.swagger.annotations.Api; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.web.bind.annotation.*; + +@RestController +@RequestMapping("/data") +@Api(tags = "data") +@Slf4j +public class DataIndexController { + @Autowired + @Qualifier("poiIndexExecutor") + private IndexExecutor indexExecutor; + + @PostMapping(value = "/index") + public ResultData indexData() { + log.info("Start to index data"); + indexExecutor.start(); + ResultData msg = new ResultData(); + msg.setMsg("ok"); + return msg; + } +} diff --git a/WebGisDemo/src/main/java/com/luxx/web/controller/MapController.java b/WebGisDemo/src/main/java/com/luxx/gis/controller/MapController.java similarity index 53% rename from WebGisDemo/src/main/java/com/luxx/web/controller/MapController.java rename to WebGisDemo/src/main/java/com/luxx/gis/controller/MapController.java index 0bcdcc6..1664117 100644 --- a/WebGisDemo/src/main/java/com/luxx/web/controller/MapController.java +++ b/WebGisDemo/src/main/java/com/luxx/gis/controller/MapController.java @@ -1,15 +1,11 @@ -package com.luxx.web.controller; +package com.luxx.gis.controller; -import io.swagger.annotations.Api; import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Controller; import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; import org.springframework.web.servlet.ModelAndView; -@RestController -@RequestMapping("/map") -@Api(tags = "map") +@Controller @Slf4j public class MapController { diff --git a/WebGisDemo/src/main/java/com/luxx/web/controller/StationController.java b/WebGisDemo/src/main/java/com/luxx/gis/controller/StationController.java similarity index 61% rename from WebGisDemo/src/main/java/com/luxx/web/controller/StationController.java rename to WebGisDemo/src/main/java/com/luxx/gis/controller/StationController.java index b18298c..ad386c1 100644 --- a/WebGisDemo/src/main/java/com/luxx/web/controller/StationController.java +++ b/WebGisDemo/src/main/java/com/luxx/gis/controller/StationController.java @@ -1,21 +1,21 @@ -package com.luxx.web.controller; +package com.luxx.gis.controller; import com.github.pagehelper.PageHelper; import com.github.pagehelper.PageInfo; -import com.luxx.web.model.Station; -import com.luxx.index.model.PoiData; -import com.luxx.web.model.PoiPoint; -import com.luxx.web.model.ResultData; -import com.luxx.web.model.request.DataInCircleRequest; -import com.luxx.web.model.request.DataInRectangleRequest; -import com.luxx.web.service.StationService; -import com.luxx.web.service.IndexService; +import com.luxx.gis.model.Station; +import com.luxx.gis.model.PoiData; +import com.luxx.gis.model.PoiPoint; +import com.luxx.gis.model.ResultData; +import com.luxx.gis.model.request.DataInCircleRequest; +import com.luxx.gis.model.request.DataInRectangleRequest; +import com.luxx.gis.service.StationService; +import com.luxx.gis.service.IndexSearchService; import io.swagger.annotations.Api; -import io.swagger.annotations.ApiOperation; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.*; +import javax.validation.constraints.Min; import java.util.List; @RestController @@ -27,11 +27,11 @@ public class StationController { private StationService stationService; @Autowired - private IndexService indexService; + private IndexSearchService indexSearchService; @GetMapping("/loadData") - @ApiOperation(value = "loadData", notes = "loadData") - public ResultData loadData(@RequestParam("pageNum") int pageNum, @RequestParam("pageSize") int pageSize) { + public ResultData loadData(@RequestParam(defaultValue = "1") @Min(1) int pageNum, + @RequestParam(defaultValue = "10") int pageSize) { PageHelper.startPage(pageNum, pageSize); List stationList = stationService.getStation(); PageInfo pageInfo = new PageInfo<>(stationList); @@ -41,23 +41,20 @@ public ResultData loadData(@RequestParam("pageNum") int pageNum, @RequestParam(" return msg; } - @RequestMapping(value = "/dataInCircle", method = RequestMethod.POST) - @ApiOperation(value = "dataInCircle", notes = "dataInCircle") + @PostMapping("/dataInCircle") public ResultData getDataInCircle(@RequestBody DataInCircleRequest request) { log.info("Query data in circle: " + request); double radius = request.getRadius(); double lat = request.getLat(); double lng = request.getLng(); - List dataList = indexService.searchPoiInCircle(lng, lat, radius); + List dataList = indexSearchService.searchPoiInCircle(lng, lat, radius); ResultData msg = new ResultData(); msg.setMsg("ok"); msg.setData(dataList); - return msg; } - @RequestMapping(value = "/dataInRectangle", method = RequestMethod.POST) - @ApiOperation(value = "dataInRectangle", notes = "dataInRectangle") + @PostMapping("/dataInRectangle") public ResultData getDataInRectangle(@RequestBody DataInRectangleRequest request) { log.info("Query data in rectangle: " + request); List points = request.getPoints(); @@ -65,19 +62,18 @@ public ResultData getDataInRectangle(@RequestBody DataInRectangleRequest request if (points != null && points.size() >= 4) { double minLat = points.get(0).getLat(); double maxLat = points.get(1).getLat(); - double minLng = points.get(0).getLng(); - double maxLng = points.get(1).getLng(); + double minLng = points.get(0).getLon(); + double maxLng = points.get(1).getLon(); for (PoiPoint poiPoint : points) { double lat = poiPoint.getLat(); - double lng = poiPoint.getLng(); + double lng = poiPoint.getLon(); maxLat = Math.max(maxLat, lat); minLat = Math.min(minLat, lat); maxLng = Math.max(maxLng, lng); minLng = Math.min(minLng, lng); } - - List dataList = indexService.searchPoiInRectangle(minLng, minLat, maxLng, maxLat); + List dataList = indexSearchService.searchPoiInRectangle(minLng, minLat, maxLng, maxLat); msg.setMsg("ok"); msg.setData(dataList); } else { diff --git a/WebGisDemo/src/main/java/com/luxx/web/dao/StationMapper.java b/WebGisDemo/src/main/java/com/luxx/gis/dao/StationMapper.java similarity index 79% rename from WebGisDemo/src/main/java/com/luxx/web/dao/StationMapper.java rename to WebGisDemo/src/main/java/com/luxx/gis/dao/StationMapper.java index 23178f5..be5adeb 100644 --- a/WebGisDemo/src/main/java/com/luxx/web/dao/StationMapper.java +++ b/WebGisDemo/src/main/java/com/luxx/gis/dao/StationMapper.java @@ -1,6 +1,6 @@ -package com.luxx.web.dao; +package com.luxx.gis.dao; -import com.luxx.web.model.Station; +import com.luxx.gis.model.Station; import org.apache.ibatis.annotations.Mapper; import java.util.List; diff --git a/IndexSearchService/src/main/java/com/luxx/index/executor/DataIndexExecutor.java b/WebGisDemo/src/main/java/com/luxx/gis/executor/IndexExecutor.java similarity index 89% rename from IndexSearchService/src/main/java/com/luxx/index/executor/DataIndexExecutor.java rename to WebGisDemo/src/main/java/com/luxx/gis/executor/IndexExecutor.java index 6a3f700..cd2da5e 100644 --- a/IndexSearchService/src/main/java/com/luxx/index/executor/DataIndexExecutor.java +++ b/WebGisDemo/src/main/java/com/luxx/gis/executor/IndexExecutor.java @@ -1,10 +1,10 @@ -package com.luxx.index.executor; +package com.luxx.gis.executor; import java.sql.Connection; import java.sql.ResultSet; import java.sql.Statement; -public abstract class DataIndexExecutor { +public abstract class IndexExecutor { public abstract void start(); public abstract void stop(); diff --git a/IndexSearchService/src/main/java/com/luxx/index/executor/MysqlToEsExecutor.java b/WebGisDemo/src/main/java/com/luxx/gis/executor/MysqlIndexExecutor.java similarity index 89% rename from IndexSearchService/src/main/java/com/luxx/index/executor/MysqlToEsExecutor.java rename to WebGisDemo/src/main/java/com/luxx/gis/executor/MysqlIndexExecutor.java index 545b8c5..2728638 100644 --- a/IndexSearchService/src/main/java/com/luxx/index/executor/MysqlToEsExecutor.java +++ b/WebGisDemo/src/main/java/com/luxx/gis/executor/MysqlIndexExecutor.java @@ -1,4 +1,4 @@ -package com.luxx.index.executor; +package com.luxx.gis.executor; import java.sql.Connection; import java.sql.ResultSet; @@ -8,13 +8,13 @@ import java.util.Date; import java.util.List; -import com.luxx.index.config.IndexConfig; -import com.luxx.index.model.HotspotData; -import com.luxx.index.service.MysqlIndexService; +import com.luxx.gis.config.IndexConfig; +import com.luxx.gis.model.HotspotData; +import com.luxx.gis.service.IndexDataSource; +import com.luxx.gis.service.MysqlIndexService; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import com.luxx.index.service.IndexDataSource; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Component; @@ -23,8 +23,8 @@ @Component @ConditionalOnProperty(name = "index.es.cluster.address") -public class MysqlToEsExecutor extends DataIndexExecutor { - private static Logger log = LogManager.getLogger(MysqlToEsExecutor.class); +public class MysqlIndexExecutor extends IndexExecutor { + private static Logger log = LogManager.getLogger(MysqlIndexExecutor.class); @Autowired private MysqlIndexService mysqlIndexService; @@ -52,7 +52,7 @@ public void start() { //indexService.deleteIndex(); mysqlIndexService.createIndex(); - mysqlIndexService.defineIndexTypeMapping(); + mysqlIndexService.createIndexMapping(); Thread exportThread = new Thread(new Runnable() { public void run() { @@ -116,6 +116,5 @@ private List getDataFromDataBase() { @Override public void stop() { - mysqlIndexService.close(); } } diff --git a/IndexSearchService/src/main/java/com/luxx/index/executor/PoiIndexExecutor.java b/WebGisDemo/src/main/java/com/luxx/gis/executor/PoiIndexExecutor.java similarity index 83% rename from IndexSearchService/src/main/java/com/luxx/index/executor/PoiIndexExecutor.java rename to WebGisDemo/src/main/java/com/luxx/gis/executor/PoiIndexExecutor.java index 2eb9e6a..152cdec 100644 --- a/IndexSearchService/src/main/java/com/luxx/index/executor/PoiIndexExecutor.java +++ b/WebGisDemo/src/main/java/com/luxx/gis/executor/PoiIndexExecutor.java @@ -1,28 +1,26 @@ -package com.luxx.index.executor; +package com.luxx.gis.executor; -import com.luxx.index.config.IndexConfig; -import com.luxx.index.model.PoiData; -import com.luxx.index.service.PoiIndexService; -import com.luxx.index.service.IndexDataSource; - -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.ArrayList; -import java.util.List; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; +import com.luxx.gis.config.IndexConfig; +import com.luxx.gis.model.PoiData; +import com.luxx.gis.service.IndexDataSource; +import com.luxx.gis.service.PoiIndexService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Component; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import javax.annotation.PostConstruct; +import java.util.List; +import java.util.ArrayList; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; @Component @ConditionalOnProperty(name = "index.db.url") -public class PoiIndexExecutor extends DataIndexExecutor { +public class PoiIndexExecutor extends IndexExecutor { private static Logger log = LogManager.getLogger(PoiIndexExecutor.class); @Autowired @@ -49,6 +47,8 @@ public void start() { log.info("Start index POI"); try { poiIndexService.clear(); + poiIndexService.createIndexMapping(); + poiIndexService.createIndex(); Thread exportThread = new Thread(new Runnable() { public void run() { @@ -87,11 +87,11 @@ private List getDataFromDataBase() { startNum, pageCount)); if (res != null) { while (res.next()) { - PoiData data = new PoiData(); - data.setId(res.getInt("id")); - data.setAddress(res.getString("address")); - data.setLat(res.getDouble("latitude")); - data.setLng(res.getDouble("longitude")); + int id = res.getInt("id"); + String address = res.getString("address"); + double lat = res.getDouble("latitude"); + double lng = res.getDouble("longitude"); + PoiData data = new PoiData(id, address, lat, lng); dataList.add(data); } } @@ -114,8 +114,5 @@ private List getDataFromDataBase() { @Override public void stop() { - if (poiIndexService != null) { - poiIndexService.close(); - } } } diff --git a/IndexSearchService/src/main/java/com/luxx/index/executor/RedshiftToEsExecutor.java b/WebGisDemo/src/main/java/com/luxx/gis/executor/RedshiftIndexExecutor.java similarity index 90% rename from IndexSearchService/src/main/java/com/luxx/index/executor/RedshiftToEsExecutor.java rename to WebGisDemo/src/main/java/com/luxx/gis/executor/RedshiftIndexExecutor.java index 6846229..d2c333d 100644 --- a/IndexSearchService/src/main/java/com/luxx/index/executor/RedshiftToEsExecutor.java +++ b/WebGisDemo/src/main/java/com/luxx/gis/executor/RedshiftIndexExecutor.java @@ -1,4 +1,4 @@ -package com.luxx.index.executor; +package com.luxx.gis.executor; import java.sql.Connection; import java.sql.ResultSet; @@ -7,10 +7,10 @@ import java.util.ArrayList; import java.util.List; -import com.luxx.index.config.IndexConfig; -import com.luxx.index.model.EndpointData; -import com.luxx.index.service.RedshiftIndexService; -import com.luxx.index.service.IndexDataSource; +import com.luxx.gis.config.IndexConfig; +import com.luxx.gis.model.EndpointData; +import com.luxx.gis.service.RedshiftIndexService; +import com.luxx.gis.service.IndexDataSource; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.springframework.beans.factory.annotation.Autowired; @@ -21,8 +21,8 @@ @Component @ConditionalOnProperty(name = "index.es.cluster.address") -public class RedshiftToEsExecutor extends DataIndexExecutor { - private static Logger log = LogManager.getLogger(RedshiftToEsExecutor.class); +public class RedshiftIndexExecutor extends IndexExecutor { + private static Logger log = LogManager.getLogger(RedshiftIndexExecutor.class); @Autowired private RedshiftIndexService indexService; @@ -53,7 +53,7 @@ public void start() { indexService.createIndex(); log.info("Create new index type mapping"); - indexService.defineIndexTypeMapping(); + indexService.createIndexMapping(); log.info("Start index redshift data to ES"); Thread exportThread = new Thread(new Runnable() { @@ -122,6 +122,5 @@ private List getDataFromDataBase() { @Override public void stop() { - indexService.close(); } } \ No newline at end of file diff --git a/IndexSearchService/src/main/java/com/luxx/index/model/EndpointData.java b/WebGisDemo/src/main/java/com/luxx/gis/model/EndpointData.java similarity index 98% rename from IndexSearchService/src/main/java/com/luxx/index/model/EndpointData.java rename to WebGisDemo/src/main/java/com/luxx/gis/model/EndpointData.java index 25bb7db..e907728 100644 --- a/IndexSearchService/src/main/java/com/luxx/index/model/EndpointData.java +++ b/WebGisDemo/src/main/java/com/luxx/gis/model/EndpointData.java @@ -1,4 +1,4 @@ -package com.luxx.index.model; +package com.luxx.gis.model; import java.io.Serializable; import java.util.Date; diff --git a/WebGisDemo/src/main/java/com/luxx/gis/model/HotspotData.java b/WebGisDemo/src/main/java/com/luxx/gis/model/HotspotData.java new file mode 100644 index 0000000..807407b --- /dev/null +++ b/WebGisDemo/src/main/java/com/luxx/gis/model/HotspotData.java @@ -0,0 +1,26 @@ +package com.luxx.gis.model; + +import lombok.Data; + +import java.io.Serializable; +import java.util.Date; + +@Data +public class HotspotData implements Serializable { + private static final long serialVersionUID = -2132799122365334550L; + + private Long id; + private int seqNum; + private Date collectTime; + + private String imsi; + private String imei; + private String tmsi; + + private String srcLac; + private String deviceID; + private String teleOper; + private String ownArea; + private String teleSevenNum; + private String teleBrand; +} diff --git a/WebGisDemo/src/main/java/com/luxx/gis/model/PoiData.java b/WebGisDemo/src/main/java/com/luxx/gis/model/PoiData.java new file mode 100644 index 0000000..d1b6235 --- /dev/null +++ b/WebGisDemo/src/main/java/com/luxx/gis/model/PoiData.java @@ -0,0 +1,24 @@ +package com.luxx.gis.model; + +import lombok.Data; + +import java.io.Serializable; + +@Data +public class PoiData implements Serializable { + private static final long serialVersionUID = -3978597686509612192L; + + private long id; + private String address; + private PoiPoint poiPoint; + + public PoiData() { + } + + public PoiData(long id, String address, double lat, double lng) { + this.id = id; + this.address = address; + this.poiPoint = new PoiPoint(lat, lng); + } + +} diff --git a/WebGisDemo/src/main/java/com/luxx/gis/model/PoiPoint.java b/WebGisDemo/src/main/java/com/luxx/gis/model/PoiPoint.java new file mode 100644 index 0000000..3c5102b --- /dev/null +++ b/WebGisDemo/src/main/java/com/luxx/gis/model/PoiPoint.java @@ -0,0 +1,17 @@ +package com.luxx.gis.model; + +import lombok.Data; + +@Data +public class PoiPoint { + private double lat; + private double lon; + + public PoiPoint() { + } + + public PoiPoint(double lat, double lon) { + this.lat = lat; + this.lon = lon; + } +} diff --git a/WebGisDemo/src/main/java/com/luxx/web/model/ResultData.java b/WebGisDemo/src/main/java/com/luxx/gis/model/ResultData.java similarity index 95% rename from WebGisDemo/src/main/java/com/luxx/web/model/ResultData.java rename to WebGisDemo/src/main/java/com/luxx/gis/model/ResultData.java index e7931b3..e7bd7bc 100644 --- a/WebGisDemo/src/main/java/com/luxx/web/model/ResultData.java +++ b/WebGisDemo/src/main/java/com/luxx/gis/model/ResultData.java @@ -1,4 +1,4 @@ -package com.luxx.web.model; +package com.luxx.gis.model; public class ResultData { private String msg; diff --git a/WebGisDemo/src/main/java/com/luxx/gis/model/Station.java b/WebGisDemo/src/main/java/com/luxx/gis/model/Station.java new file mode 100644 index 0000000..a8a5ea4 --- /dev/null +++ b/WebGisDemo/src/main/java/com/luxx/gis/model/Station.java @@ -0,0 +1,22 @@ +package com.luxx.gis.model; + +import lombok.Data; + +@Data +public class Station { + private long id; + + private String countryCode; + + private String provider; + + private String lac; + + private String cell; + + private double latitude; + + private double longitude; + + private String address; +} diff --git a/WebGisDemo/src/main/java/com/luxx/gis/model/request/DataInCircleRequest.java b/WebGisDemo/src/main/java/com/luxx/gis/model/request/DataInCircleRequest.java new file mode 100644 index 0000000..21742ad --- /dev/null +++ b/WebGisDemo/src/main/java/com/luxx/gis/model/request/DataInCircleRequest.java @@ -0,0 +1,10 @@ +package com.luxx.gis.model.request; + +import lombok.Data; + +@Data +public class DataInCircleRequest { + private double radius; + private double lat; + private double lng; +} diff --git a/WebGisDemo/src/main/java/com/luxx/gis/model/request/DataInRectangleRequest.java b/WebGisDemo/src/main/java/com/luxx/gis/model/request/DataInRectangleRequest.java new file mode 100644 index 0000000..0423ca5 --- /dev/null +++ b/WebGisDemo/src/main/java/com/luxx/gis/model/request/DataInRectangleRequest.java @@ -0,0 +1,11 @@ +package com.luxx.gis.model.request; + +import com.luxx.gis.model.PoiPoint; +import lombok.Data; + +import java.util.List; + +@Data +public class DataInRectangleRequest { + private List points; +} diff --git a/IndexSearchService/src/main/java/com/luxx/index/service/IndexDataSource.java b/WebGisDemo/src/main/java/com/luxx/gis/service/IndexDataSource.java similarity index 96% rename from IndexSearchService/src/main/java/com/luxx/index/service/IndexDataSource.java rename to WebGisDemo/src/main/java/com/luxx/gis/service/IndexDataSource.java index 74d9877..4b49737 100644 --- a/IndexSearchService/src/main/java/com/luxx/index/service/IndexDataSource.java +++ b/WebGisDemo/src/main/java/com/luxx/gis/service/IndexDataSource.java @@ -1,9 +1,9 @@ -package com.luxx.index.service; +package com.luxx.gis.service; import java.sql.Connection; import java.sql.SQLException; -import com.luxx.index.config.IndexConfig; +import com.luxx.gis.config.IndexConfig; import com.zaxxer.hikari.HikariConfig; import com.zaxxer.hikari.HikariDataSource; import org.springframework.beans.factory.annotation.Autowired; diff --git a/WebGisDemo/src/main/java/com/luxx/web/service/IndexService.java b/WebGisDemo/src/main/java/com/luxx/gis/service/IndexSearchService.java similarity index 58% rename from WebGisDemo/src/main/java/com/luxx/web/service/IndexService.java rename to WebGisDemo/src/main/java/com/luxx/gis/service/IndexSearchService.java index f66ca4b..4b2089b 100644 --- a/WebGisDemo/src/main/java/com/luxx/web/service/IndexService.java +++ b/WebGisDemo/src/main/java/com/luxx/gis/service/IndexSearchService.java @@ -1,7 +1,6 @@ -package com.luxx.web.service; +package com.luxx.gis.service; -import com.luxx.index.model.PoiData; -import com.luxx.index.service.PoiIndexService; +import com.luxx.gis.model.PoiData; import org.springframework.beans.factory.DisposableBean; import org.springframework.beans.factory.InitializingBean; import org.springframework.beans.factory.annotation.Autowired; @@ -11,24 +10,12 @@ import java.util.List; @Service -@Lazy -public class IndexService implements InitializingBean, DisposableBean { +public class IndexSearchService { @Autowired private PoiIndexService poiIndexService; - @Override - public void destroy() throws Exception { - if (poiIndexService != null) { - poiIndexService.close(); - } - } - - @Override - public void afterPropertiesSet() throws Exception { - } - public List searchPoiInCircle(double lng, double lat, double radius) { - return poiIndexService.searchPoiInCircle(lng, lat, radius / 1000); + return poiIndexService.searchPoiInCircle(lng, lat, String.valueOf(radius)); } public List searchPoiInRectangle(double minLng, double minLat, diff --git a/IndexSearchService/src/main/java/com/luxx/index/service/MysqlIndexService.java b/WebGisDemo/src/main/java/com/luxx/gis/service/MysqlIndexService.java similarity index 62% rename from IndexSearchService/src/main/java/com/luxx/index/service/MysqlIndexService.java rename to WebGisDemo/src/main/java/com/luxx/gis/service/MysqlIndexService.java index a637a26..db8e437 100644 --- a/IndexSearchService/src/main/java/com/luxx/index/service/MysqlIndexService.java +++ b/WebGisDemo/src/main/java/com/luxx/gis/service/MysqlIndexService.java @@ -1,21 +1,15 @@ -package com.luxx.index.service; +package com.luxx.gis.service; import java.io.IOException; import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Map; -import com.fasterxml.jackson.core.JsonParseException; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.JsonMappingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.luxx.index.client.ElasticSearchClient; -import com.luxx.index.model.HotspotData; +import com.luxx.gis.client.ElasticSearchClient; +import com.luxx.gis.model.HotspotData; -import org.elasticsearch.action.bulk.BulkItemResponse; -import org.elasticsearch.action.bulk.BulkRequestBuilder; -import org.elasticsearch.action.bulk.BulkResponse; +import com.luxx.gis.util.JsonUtil; +import org.elasticsearch.action.bulk.BulkProcessor; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.query.QueryBuilder; @@ -41,8 +35,6 @@ public class MysqlIndexService { // Index Name private static final String Index = "hotspot"; - // Type Name - private static final String Type = "hotspotdata"; // Field Name private static final String TimeFieldName = "CollectTime"; @@ -56,10 +48,6 @@ public class MysqlIndexService { private static final String DateFormat = "yyyy-MM-dd HH:mm:ss"; - public void close() { - elasticSearchClient.close(); - } - public void deleteIndex() { elasticSearchClient.deleteIndex(Index); } @@ -68,84 +56,62 @@ public void createIndex() { elasticSearchClient.createIndex(Index); } - public void defineIndexTypeMapping() { - XContentBuilder mapBuilder = prepareMappingBuilder(); - elasticSearchClient.defineIndexTypeMapping(Index, Type, mapBuilder); + public void createIndexMapping() { + XContentBuilder contentBuilder = prepareMappingBuilder(); + elasticSearchClient.createIndexMapping(Index, contentBuilder); } public XContentBuilder prepareMappingBuilder() { - XContentBuilder mapBuilder = null; + XContentBuilder contentBuilder = null; try { - mapBuilder = XContentFactory.jsonBuilder(); - mapBuilder.startObject().startObject(Type).startObject("properties").startObject(IDFieldName) - .field("type", "long").field("include_in_all", "false").endObject().startObject(SeqNumFieldName) - .field("type", "long").field("include_in_all", "false").endObject().startObject(IMSIFieldName) - .field("type", "text").field("index", "not_analyzed").field("include_in_all", "false").endObject() - .startObject(IMEIFieldName).field("type", "text").field("index", "not_analyzed") - .field("include_in_all", "false").endObject().startObject(DeviceIDFieldName).field("type", "text") - .field("index", "not_analyzed").field("include_in_all", "false").endObject() - .startObject(OwnAreaFieldName).field("type", "text").field("index", "not_analyzed") - .field("include_in_all", "false").endObject().startObject(TeleOperFieldName).field("type", "text") - .field("index", "not_analyzed").field("include_in_all", "false").endObject() - .startObject(TimeFieldName).field("type", "date").field("format", DateFormat) - .field("include_in_all", "false").endObject().endObject().endObject().endObject(); + contentBuilder = XContentFactory.jsonBuilder(); + contentBuilder.startObject(); + { + contentBuilder.startObject("properties"); + { + contentBuilder.startObject(IDFieldName).field("type", "long").endObject(); + contentBuilder.startObject(SeqNumFieldName).field("type", "long").endObject(); + contentBuilder.startObject(IMSIFieldName).field("type", "keyword").endObject(); + contentBuilder.startObject(IMEIFieldName).field("type", "keyword").endObject(); + contentBuilder.startObject(DeviceIDFieldName).field("type", "keyword").endObject(); + contentBuilder.startObject(OwnAreaFieldName).field("type", "keyword").endObject(); + contentBuilder.startObject(TeleOperFieldName).field("type", "keyword").endObject(); + contentBuilder.startObject(TimeFieldName).field("type", "date").field("format", DateFormat).endObject(); + } + contentBuilder.endObject(); + } + contentBuilder.endObject(); } catch (IOException e) { log.error(e); } - - return mapBuilder; - } - - //https://stackoverflow.com/questions/27427613/elasticsearch-java-api-putmapping-from-json-file-error - public static XContentBuilder builderFromJson(String json) throws JsonParseException, JsonMappingException, IOException { - Map map = new ObjectMapper().readValue(json, new TypeReference>() { - }); - return XContentFactory.jsonBuilder().map(map); + return contentBuilder; } // Build json data public String getIndexDataFromHotspotData(HotspotData data) { - String jsonString = null; + Map map = new HashMap<>(); if (data != null) { - try { - XContentBuilder jsonBuilder = XContentFactory.jsonBuilder(); - jsonBuilder.startObject().field(IDFieldName, data.getId()).field(SeqNumFieldName, data.getSeqNum()) - .field(IMSIFieldName, data.getImsi()).field(IMEIFieldName, data.getImei()) - .field(DeviceIDFieldName, data.getDeviceID()).field(OwnAreaFieldName, data.getOwnArea()) - .field(TeleOperFieldName, data.getTeleOper()).field(TimeFieldName, data.getCollectTime()) - .endObject(); - jsonString = jsonBuilder.string(); - } catch (IOException e) { - log.error(e); - } + map.put(IDFieldName, data.getId()); + map.put(SeqNumFieldName, data.getSeqNum()); + map.put(IMSIFieldName, data.getImsi()); + map.put(IMEIFieldName, data.getImei()); + map.put(DeviceIDFieldName, data.getDeviceID()); + map.put(OwnAreaFieldName, data.getOwnArea()); + map.put(TeleOperFieldName, data.getTeleOper()); + map.put(TimeFieldName, data.getCollectTime()); } - - return jsonString; + return JsonUtil.objectToJson(map); } // Index data in bulk public void indexHotSpotDataList(List dataList) { - if (dataList != null) { - int size = dataList.size(); - if (size > 0) { - BulkRequestBuilder bulkRequest = elasticSearchClient.getBulkRequest(); - for (int i = 0; i < size; ++i) { - HotspotData data = dataList.get(i); - String jsonSource = getIndexDataFromHotspotData(data); - if (jsonSource != null) { - bulkRequest.add(elasticSearchClient.getIndexRequest(Index, Type, jsonSource)); - } - } - - BulkResponse bulkResponse = bulkRequest.execute().actionGet(); - if (bulkResponse.hasFailures()) { - Iterator iter = bulkResponse.iterator(); - while (iter.hasNext()) { - BulkItemResponse itemResponse = iter.next(); - if (itemResponse.isFailed()) { - log.error(itemResponse.getFailureMessage()); - } - } + if (dataList != null && dataList.size() > 0) { + BulkProcessor bulkRequest = elasticSearchClient.getBulkRequest(); + for (int i = 0; i < dataList.size(); ++i) { + HotspotData data = dataList.get(i); + String jsonSource = getIndexDataFromHotspotData(data); + if (jsonSource != null) { + bulkRequest.add(elasticSearchClient.getIndexRequest(Index, jsonSource)); } } } @@ -166,43 +132,41 @@ private QueryBuilder getDateRangeQueryBuilder(String startTime, String endTime) if (startTime != null && endTime != null) { rangeBuilder = QueryBuilders.rangeQuery(TimeFieldName).from(startTime).to(endTime).format(DateFormat); } - return rangeBuilder; } // Get result from startTime to endTime public List getSearchResultBetweenTime(String startTime, String endTime) { QueryBuilder rangeBuilder = getDateRangeQueryBuilder(startTime, endTime); - return elasticSearchClient.getSearchDataByScrolls(Index, Type, rangeBuilder, 1000000); + return elasticSearchClient.getSearchDataByScrolls(Index, rangeBuilder); } // Get count histogram based on date from startTime to endTime public Map getDeviceDateFacetDistributeInfo(String startTime, String endTime, List deviceList, final String gap) { - Map resultsMap = new HashMap<>(); QueryBuilder deviceQueryBuilder = getDeviceQueryBuilder(deviceList); QueryBuilder rangeBuilder = getDateRangeQueryBuilder(startTime, endTime); QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(deviceQueryBuilder).must(rangeBuilder); DateHistogramAggregationBuilder dateHistogramAggBuilder = AggregationBuilders.dateHistogram("dateAgg").field(TimeFieldName) - .dateHistogramInterval(DateHistogramInterval.YEAR); + .fixedInterval(DateHistogramInterval.YEAR); switch (gap) { case "Quarter": dateHistogramAggBuilder = AggregationBuilders.dateHistogram("dateAgg").field(TimeFieldName) - .dateHistogramInterval(DateHistogramInterval.QUARTER); + .fixedInterval(DateHistogramInterval.QUARTER); break; case "Month": dateHistogramAggBuilder = AggregationBuilders.dateHistogram("dateAgg").field(TimeFieldName) - .dateHistogramInterval(DateHistogramInterval.MONTH); + .fixedInterval(DateHistogramInterval.MONTH); break; case "Week": dateHistogramAggBuilder = AggregationBuilders.dateHistogram("dateAgg").field(TimeFieldName) - .dateHistogramInterval(DateHistogramInterval.WEEK); + .fixedInterval(DateHistogramInterval.WEEK); break; case "Day": dateHistogramAggBuilder = AggregationBuilders.dateHistogram("dateAgg").field(TimeFieldName) - .dateHistogramInterval(DateHistogramInterval.DAY); + .fixedInterval(DateHistogramInterval.DAY); break; } @@ -212,7 +176,6 @@ public Map getDeviceDateFacetDistributeInfo(String startTime, St // Get count aggregation based on device from startTime to endTime public Map getDeviceDistributedInfo(String startTime, String endTime, List deviceList) { - Map resultsMap = new HashMap<>(); QueryBuilder deviceQueryBuilder = getDeviceQueryBuilder(deviceList); diff --git a/WebGisDemo/src/main/java/com/luxx/gis/service/PoiIndexService.java b/WebGisDemo/src/main/java/com/luxx/gis/service/PoiIndexService.java new file mode 100644 index 0000000..ffe9f6e --- /dev/null +++ b/WebGisDemo/src/main/java/com/luxx/gis/service/PoiIndexService.java @@ -0,0 +1,187 @@ +package com.luxx.gis.service; + +import com.luxx.gis.client.ElasticSearchClient; +import com.luxx.gis.model.PoiData; +import com.luxx.gis.util.JsonUtil; +import org.apache.lucene.document.*; +import org.apache.lucene.search.*; +import org.apache.lucene.search.BooleanClause.Occur; +import org.apache.lucene.spatial.query.SpatialArgs; +import org.apache.lucene.spatial.query.SpatialOperation; +import org.apache.lucene.util.QueryBuilder; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.bulk.BulkProcessor; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.common.unit.DistanceUnit; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.query.GeoBoundingBoxQueryBuilder; +import org.elasticsearch.index.query.GeoDistanceQueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.query.TermQueryBuilder; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import javax.swing.text.html.HTMLDocument; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +@Service +public class PoiIndexService { + private static Logger log = LogManager.getLogger(PoiIndexService.class); + + // Field Name + private static final String IDFieldName = "id"; + private static final String AddressFieldName = "address"; + private static final String LatFieldName = "lat"; + private static final String LngFieldName = "lng"; + private static final String LocationFieldName = "poiPoint"; + + private static final String IndexName = "geo_location"; + + private final int maxResultCount = 100; + + @Autowired + private ElasticSearchClient elasticSearchClient; + + public void clear() { + elasticSearchClient.deleteIndex(IndexName); + } + + public void createIndex() { + elasticSearchClient.createIndex(IndexName); + } + + public void createIndexMapping() { + XContentBuilder contentBuilder = prepareMappingBuilder(); + elasticSearchClient.createIndexMapping(IndexName, contentBuilder); + } + + public XContentBuilder prepareMappingBuilder() { + XContentBuilder contentBuilder = null; + try { + contentBuilder = XContentFactory.jsonBuilder(); + contentBuilder.startObject(); + { + contentBuilder.startObject("properties"); + { + contentBuilder.startObject(IDFieldName).field("type", "long").endObject(); + contentBuilder.startObject(AddressFieldName).field("type", "keyword").endObject(); + contentBuilder.startObject(LocationFieldName).field("type", "geo_point").endObject(); + } + contentBuilder.endObject(); + } + contentBuilder.endObject(); + } catch (IOException e) { + log.error(e); + } + return contentBuilder; + } + + public boolean indexPoiDataList(List dataList) { + try { + if (dataList != null && dataList.size() > 0) { + BulkProcessor bulkRequest = elasticSearchClient.getBulkRequest(); + for (int i = 0; i < dataList.size(); ++i) { + PoiData data = dataList.get(i); + String jsonSource = JsonUtil.objectToJson(data); + if (jsonSource != null) { + bulkRequest.add(elasticSearchClient.getIndexRequest(IndexName, jsonSource)); + } + } + return true; + } + return false; + } catch (Exception e) { + log.error(e.toString()); + return false; + } + } + + public List searchPoiInRectangle(double minLng, double minLat, double maxLng, double maxLat) { + List results = new ArrayList<>(); + SearchRequest searchRequest = new SearchRequest(IndexName); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + GeoBoundingBoxQueryBuilder geoBoundingBoxQueryBuilder = QueryBuilders.geoBoundingBoxQuery(LocationFieldName) + .setCorners(maxLat, minLng, minLat, maxLng); + sourceBuilder.query(geoBoundingBoxQueryBuilder); + searchRequest.source(sourceBuilder); + try { + SearchResponse searchResponse = elasticSearchClient.getClient().search(searchRequest, RequestOptions.DEFAULT); + SearchHits searchHits = searchResponse.getHits(); + for (SearchHit searchHit : searchHits) { + Long id = (Long) searchHit.getSourceAsMap().get(IDFieldName); + String address = (String) searchHit.getSourceAsMap().get(AddressFieldName); + GeoPoint geoPoint = (GeoPoint) searchHit.getSourceAsMap().get(LocationFieldName); + PoiData poiData = new PoiData(id, address, geoPoint.getLat(), geoPoint.getLon()); + results.add(poiData); + } + } catch (Exception e) { + log.error("Search Poi in rectangle error: " + e.toString()); + } + return results; + } + + public List searchPoiInCircle(double lng, double lat, String distanceKm) { + List results = new ArrayList<>(); + SearchRequest searchRequest = new SearchRequest(IndexName); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + GeoDistanceQueryBuilder geoDistanceQueryBuilder = QueryBuilders.geoDistanceQuery(LocationFieldName) + .distance(distanceKm, DistanceUnit.KILOMETERS) + .point(lat, lng); + sourceBuilder.query(geoDistanceQueryBuilder); + searchRequest.source(sourceBuilder); + try { + SearchResponse searchResponse = elasticSearchClient.getClient().search(searchRequest, RequestOptions.DEFAULT); + SearchHits searchHits = searchResponse.getHits(); + for (SearchHit searchHit : searchHits) { + Long id = (Long) searchHit.getSourceAsMap().get(IDFieldName); + String address = (String) searchHit.getSourceAsMap().get(AddressFieldName); + GeoPoint geoPoint = (GeoPoint) searchHit.getSourceAsMap().get(LocationFieldName); + PoiData poiData = new PoiData(id, address, geoPoint.getLat(), geoPoint.getLon()); + results.add(poiData); + } + } catch (Exception e) { + log.error("Search Poi in circle error: " + e.toString()); + } + + return results; + } + + public List searchPoiInCircleAndAddress(double lng, double lat, String distanceKm, String address) { + List results = new ArrayList<>(); + SearchRequest searchRequest = new SearchRequest(IndexName); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + GeoDistanceQueryBuilder geoDistanceQueryBuilder = QueryBuilders.geoDistanceQuery(LocationFieldName) + .distance(distanceKm, DistanceUnit.KILOMETERS) + .point(lat, lng); + TermQueryBuilder termQueryBuilder = QueryBuilders.termQuery(AddressFieldName, address); + sourceBuilder.query(geoDistanceQueryBuilder).query(termQueryBuilder); + searchRequest.source(sourceBuilder); + try { + SearchResponse searchResponse = elasticSearchClient.getClient().search(searchRequest, RequestOptions.DEFAULT); + SearchHits searchHits = searchResponse.getHits(); + for (SearchHit searchHit : searchHits) { + Long id = (Long) searchHit.getSourceAsMap().get(IDFieldName); + String addr = (String) searchHit.getSourceAsMap().get(AddressFieldName); + GeoPoint geoPoint = (GeoPoint) searchHit.getSourceAsMap().get(LocationFieldName); + PoiData poiData = new PoiData(id, addr, geoPoint.getLat(), geoPoint.getLon()); + results.add(poiData); + } + } catch (Exception e) { + log.error("Search Poi in circle and address error: " + e.toString()); + } + + return results; + } + +} diff --git a/IndexSearchService/src/main/java/com/luxx/index/service/RedshiftIndexService.java b/WebGisDemo/src/main/java/com/luxx/gis/service/RedshiftIndexService.java similarity index 52% rename from IndexSearchService/src/main/java/com/luxx/index/service/RedshiftIndexService.java rename to WebGisDemo/src/main/java/com/luxx/gis/service/RedshiftIndexService.java index 2cbf8ac..a1c3e3b 100644 --- a/IndexSearchService/src/main/java/com/luxx/index/service/RedshiftIndexService.java +++ b/WebGisDemo/src/main/java/com/luxx/gis/service/RedshiftIndexService.java @@ -1,18 +1,16 @@ -package com.luxx.index.service; +package com.luxx.gis.service; import java.io.IOException; import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Map; -import com.luxx.index.client.ElasticSearchClient; -import com.luxx.index.model.EndpointData; +import com.luxx.gis.client.ElasticSearchClient; +import com.luxx.gis.model.EndpointData; +import com.luxx.gis.util.JsonUtil; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.action.bulk.BulkItemResponse; -import org.elasticsearch.action.bulk.BulkRequestBuilder; -import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.bulk.*; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.query.QueryBuilder; @@ -35,17 +33,11 @@ public class RedshiftIndexService { // Index Name private static final String Index = "endpoint"; - // Type Name - private static final String Type = "endpointdata"; private static final String DateFormat = "yyyy-MM-dd HH:mm:ss"; private static final String TimeField = "date_time"; - public void close() { - elasticSearchClient.close(); - } - public void deleteIndex() { elasticSearchClient.deleteIndex(Index); } @@ -54,78 +46,68 @@ public void createIndex() { elasticSearchClient.createIndex(Index); } - public void defineIndexTypeMapping() { - XContentBuilder mapBuilder = prepareMappingBuilder(); - elasticSearchClient.defineIndexTypeMapping(Index, Type, mapBuilder); + public void createIndexMapping() { + XContentBuilder contentBuilder = prepareMappingBuilder(); + elasticSearchClient.createIndexMapping(Index, contentBuilder); } public XContentBuilder prepareMappingBuilder() { - XContentBuilder mapBuilder = null; + XContentBuilder contentBuilder = null; try { - mapBuilder = XContentFactory.jsonBuilder(); - mapBuilder.startObject().startObject(Type).startObject("properties").startObject("org_id") - .field("type", "integer").field("include_in_all", "false").endObject().startObject("endpoint_id") - .field("type", "long").field("include_in_all", "false").endObject().startObject("ds_bytes") - .field("type", "long").field("include_in_all", "false").endObject().startObject("ds_max_bps") - .field("type", "double").field("include_in_all", "false").endObject().startObject("ds_avg_bps") - .field("type", "double").field("include_in_all", "false").endObject().startObject("ds_mwt") - .field("type", "integer").field("include_in_all", "false").endObject().startObject("us_bytes") - .field("type", "long").field("include_in_all", "false").endObject().startObject("us_max_bps") - .field("type", "double").field("include_in_all", "false").endObject().startObject("us_avg_bps") - .field("type", "double").field("include_in_all", "false").endObject().startObject("us_mwt") - .field("type", "integer").field("include_in_all", "false").endObject().startObject(TimeField) - .field("type", "date").field("include_in_all", "false").field("format", DateFormat).endObject().endObject() - .endObject().endObject(); + contentBuilder = XContentFactory.jsonBuilder(); + contentBuilder.startObject(); + { + contentBuilder.startObject("properties"); + { + contentBuilder.startObject("org_id").field("type", "integer").endObject(); + contentBuilder.startObject("endpoint_id").field("type", "long").endObject(); + contentBuilder.startObject("ds_bytes").field("type", "long").endObject(); + contentBuilder.startObject("ds_max_bps").field("type", "double").endObject(); + contentBuilder.startObject("ds_avg_bps").field("type", "double").endObject(); + contentBuilder.startObject("ds_mwt").field("type", "integer").endObject(); + contentBuilder.startObject("us_bytes").field("type", "long").endObject(); + contentBuilder.startObject("us_max_bps").field("type", "double").endObject(); + contentBuilder.startObject("us_avg_bps").field("type", "double").endObject(); + contentBuilder.startObject("us_mwt").field("type", "integer").endObject(); + contentBuilder.startObject(TimeField).field("type", "date").field("format", DateFormat).endObject(); + } + contentBuilder.endObject(); + } + contentBuilder.endObject(); } catch (IOException e) { log.error(e); } - return mapBuilder; + return contentBuilder; } public String getIndexDataFromHotspotDataForRedshift(EndpointData data) { - String jsonString = null; + Map map = new HashMap<>(); if (data != null) { - try { - XContentBuilder jsonBuilder = XContentFactory.jsonBuilder(); - jsonBuilder.startObject().field("org_id", data.getOrg_id()).field("endpoint_id", data.getEndpoint_id()) - .field("ds_bytes", data.getDs_bytes()).field("ds_max_bps", data.getDs_max_bytes()) - .field("ds_avg_bps", data.getDs_avg_bytes()).field("ds_mwt", data.getDs_mwt()) - .field("us_bytes", data.getUs_bytes()).field("us_max_bps", data.getUs_max_bytes()) - .field("us_avg_bps", data.getUs_avg_bytes()).field("us_mwt", data.getUs_mwt()) - .field(TimeField, data.getDate_time()).endObject(); - jsonString = jsonBuilder.string(); - } catch (IOException e) { - log.error(e); - } + map.put("org_id", data.getOrg_id()); + map.put("endpoint_id", data.getEndpoint_id()); + map.put("ds_bytes", data.getDs_bytes()); + map.put("ds_max_bps", data.getDs_max_bytes()); + map.put("ds_avg_bps", data.getDs_avg_bytes()); + map.put("ds_mwt", data.getDs_mwt()); + map.put("us_bytes", data.getUs_bytes()); + map.put("us_max_bps", data.getUs_max_bytes()); + map.put("us_avg_bps", data.getUs_avg_bytes()); + map.put("us_mwt", data.getUs_mwt()); + map.put(TimeField, data.getDate_time()); } - - return jsonString; + return JsonUtil.objectToJson(map); } // Index data in bulk public void indexHotSpotDataListForRedshift(List dataList) { - if (dataList != null) { - int size = dataList.size(); - if (size > 0) { - BulkRequestBuilder bulkRequest = elasticSearchClient.getBulkRequest(); - for (int i = 0; i < size; ++i) { - EndpointData data = dataList.get(i); - String jsonSource = getIndexDataFromHotspotDataForRedshift(data); - if (jsonSource != null) { - bulkRequest.add(elasticSearchClient.getIndexRequest(Index, Type, jsonSource)); - } - } - - BulkResponse bulkResponse = bulkRequest.execute().actionGet(); - if (bulkResponse.hasFailures()) { - Iterator iter = bulkResponse.iterator(); - while (iter.hasNext()) { - BulkItemResponse itemResponse = iter.next(); - if (itemResponse.isFailed()) { - log.error(itemResponse.getFailureMessage()); - } - } + if (dataList != null && dataList.size() > 0) { + BulkProcessor bulkRequest = elasticSearchClient.getBulkRequest(); + for (int i = 0; i < dataList.size(); ++i) { + EndpointData data = dataList.get(i); + String jsonSource = getIndexDataFromHotspotDataForRedshift(data); + if (jsonSource != null) { + bulkRequest.add(elasticSearchClient.getIndexRequest(Index, jsonSource)); } } } @@ -161,7 +143,6 @@ public Map getTopNEndpointUsage(String startTime, String endTime // Get endpoint_id and max_bps based on query from startTime to endTime public Map getEndpointMaxbps(String startTime, String endTime, int orgId) { Map resultsMap = new HashMap<>(); - QueryBuilder orgIdQueryBuilder = QueryBuilders.termQuery("org_id", orgId); QueryBuilder endpointIdFilterBuilder = QueryBuilders.termQuery("endpoint_id", 0); QueryBuilder rangeBuilder = getDateRangeQueryBuilder(startTime, endTime); @@ -172,7 +153,6 @@ public Map getEndpointMaxbps(String startTime, String endTime, i termsBuilder.subAggregation(AggregationBuilders.max("max_bps").field("ds_max_bps")); resultsMap = elasticSearchClient.getMaxAggSearchOrderResult(Index, queryBuilder, termsBuilder, "endpointMaxAgg", "max_bps"); - return resultsMap; } diff --git a/WebGisDemo/src/main/java/com/luxx/web/service/StationService.java b/WebGisDemo/src/main/java/com/luxx/gis/service/StationService.java similarity index 68% rename from WebGisDemo/src/main/java/com/luxx/web/service/StationService.java rename to WebGisDemo/src/main/java/com/luxx/gis/service/StationService.java index cf894fa..b4d9522 100644 --- a/WebGisDemo/src/main/java/com/luxx/web/service/StationService.java +++ b/WebGisDemo/src/main/java/com/luxx/gis/service/StationService.java @@ -1,9 +1,9 @@ -package com.luxx.web.service; +package com.luxx.gis.service; import java.util.List; -import com.luxx.web.dao.StationMapper; -import com.luxx.web.model.Station; +import com.luxx.gis.dao.StationMapper; +import com.luxx.gis.model.Station; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -16,17 +16,16 @@ public List getStation() { return stationMapper.getStation(); } - public int insert(Station station){ + public int insert(Station station) { return stationMapper.insert(station); } - public int update(Station station){ + public int update(Station station) { return stationMapper.update(station); } - public int delete(int id){ + public int delete(int id) { return stationMapper.delete(id); } - } diff --git a/IndexSearchService/src/main/java/com/luxx/index/util/DateFormatUtil.java b/WebGisDemo/src/main/java/com/luxx/gis/util/DateFormatUtil.java similarity index 98% rename from IndexSearchService/src/main/java/com/luxx/index/util/DateFormatUtil.java rename to WebGisDemo/src/main/java/com/luxx/gis/util/DateFormatUtil.java index 1fe496c..43e489f 100644 --- a/IndexSearchService/src/main/java/com/luxx/index/util/DateFormatUtil.java +++ b/WebGisDemo/src/main/java/com/luxx/gis/util/DateFormatUtil.java @@ -1,4 +1,4 @@ -package com.luxx.index.util; +package com.luxx.gis.util; import java.text.SimpleDateFormat; import java.util.Date; diff --git a/IndexSearchService/src/main/java/com/luxx/index/util/JsonUtil.java b/WebGisDemo/src/main/java/com/luxx/gis/util/JsonUtil.java similarity index 87% rename from IndexSearchService/src/main/java/com/luxx/index/util/JsonUtil.java rename to WebGisDemo/src/main/java/com/luxx/gis/util/JsonUtil.java index ca6179e..c17892b 100644 --- a/IndexSearchService/src/main/java/com/luxx/index/util/JsonUtil.java +++ b/WebGisDemo/src/main/java/com/luxx/gis/util/JsonUtil.java @@ -1,10 +1,9 @@ -package com.luxx.index.util; +package com.luxx.gis.util; import java.io.IOException; import java.text.SimpleDateFormat; import java.util.HashMap; -import com.fasterxml.jackson.core.JsonParser.Feature; import com.fasterxml.jackson.databind.JavaType; import com.fasterxml.jackson.databind.ObjectMapper; @@ -20,7 +19,6 @@ public class JsonUtil { public static type jsonToObject(String json, Class cls) { type obj = null; JavaType javaType = objMapper.getTypeFactory().constructType(cls); - objMapper.configure(Feature.ALLOW_MISSING_VALUES, true); try { obj = objMapper.readValue(json, javaType); } catch (IOException e) { @@ -32,7 +30,6 @@ public static type jsonToObject(String json, Class cls) { public static type jsonToObjectList(String json, Class collectionClass, Class... elementClass) { type obj = null; JavaType javaType = objMapper.getTypeFactory().constructParametricType(collectionClass, elementClass); - objMapper.configure(Feature.ALLOW_MISSING_VALUES, true); try { obj = objMapper.readValue(json, javaType); } catch (IOException e) { @@ -44,7 +41,6 @@ public static type jsonToObjectList(String json, Class collectionClass public static type jsonToObjectHashMap(String json, Class keyClass, Class valueClass) { type obj = null; JavaType javaType = objMapper.getTypeFactory().constructParametricType(HashMap.class, keyClass, valueClass); - objMapper.configure(Feature.ALLOW_MISSING_VALUES, true); try { obj = objMapper.readValue(json, javaType); } catch (IOException e) { diff --git a/WebGisDemo/src/main/java/com/luxx/web/WebApp.java b/WebGisDemo/src/main/java/com/luxx/web/WebApp.java deleted file mode 100644 index 4c7f82c..0000000 --- a/WebGisDemo/src/main/java/com/luxx/web/WebApp.java +++ /dev/null @@ -1,15 +0,0 @@ -package com.luxx.web; - -import org.springframework.boot.SpringApplication; -import org.springframework.boot.SpringBootConfiguration; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.context.annotation.ComponentScan; - -@SpringBootApplication -@SpringBootConfiguration -@ComponentScan(basePackages = "com.luxx") -public class WebApp { - public static void main(String[] args) { - SpringApplication.run(WebApp.class, args); - } -} diff --git a/WebGisDemo/src/main/java/com/luxx/web/config/WebMvcConfig.java b/WebGisDemo/src/main/java/com/luxx/web/config/WebMvcConfig.java deleted file mode 100644 index 60ef707..0000000 --- a/WebGisDemo/src/main/java/com/luxx/web/config/WebMvcConfig.java +++ /dev/null @@ -1,16 +0,0 @@ -package com.luxx.web.config; - -import org.springframework.context.annotation.Configuration; -import org.springframework.web.servlet.config.annotation.ResourceHandlerRegistry; -import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; - -// https://www.jianshu.com/p/d127c4f78bb8 -@Configuration -public class WebMvcConfig implements WebMvcConfigurer { - @Override - public void addResourceHandlers(ResourceHandlerRegistry registry) { - // Swagger ui Mapping - registry.addResourceHandler("swagger-ui.html") - .addResourceLocations("classpath:/META-INF/resources/"); - } -} diff --git a/WebGisDemo/src/main/java/com/luxx/web/model/PoiPoint.java b/WebGisDemo/src/main/java/com/luxx/web/model/PoiPoint.java deleted file mode 100644 index 5f2d87b..0000000 --- a/WebGisDemo/src/main/java/com/luxx/web/model/PoiPoint.java +++ /dev/null @@ -1,39 +0,0 @@ -package com.luxx.web.model; - -public class PoiPoint { - private double lat; - private double lng; - - public PoiPoint() { - - } - - public PoiPoint(double lat, double lng) { - this.lat = lat; - this.lng = lng; - } - - public double getLat() { - return lat; - } - - public void setLat(double lat) { - this.lat = lat; - } - - public double getLng() { - return lng; - } - - public void setLng(double lng) { - this.lng = lng; - } - - @Override - public String toString() { - return "PoiPoint{" + - "lat=" + lat + - ", lng=" + lng + - '}'; - } -} diff --git a/WebGisDemo/src/main/java/com/luxx/web/model/Station.java b/WebGisDemo/src/main/java/com/luxx/web/model/Station.java deleted file mode 100644 index 19cf5ce..0000000 --- a/WebGisDemo/src/main/java/com/luxx/web/model/Station.java +++ /dev/null @@ -1,101 +0,0 @@ -package com.luxx.web.model; - -public class Station { - private long id; - - private String countryCode; - - private String provider; - - private String lac; - - private String cell; - - private double latitude; - - private double longitude; - - private String address; - - public Station() { - - } - - public long getId() { - return id; - } - - public void setId(long id) { - this.id = id; - } - - public String getCountryCode() { - return countryCode; - } - - public void setCountryCode(String countryCode) { - this.countryCode = countryCode; - } - - public String getProvider() { - return provider; - } - - public void setProvider(String provider) { - this.provider = provider; - } - - public String getLac() { - return lac; - } - - public void setLac(String lac) { - this.lac = lac; - } - - public String getCell() { - return cell; - } - - public void setCell(String cell) { - this.cell = cell; - } - - public double getLatitude() { - return latitude; - } - - public void setLatitude(double latitude) { - this.latitude = latitude; - } - - public double getLongitude() { - return longitude; - } - - public void setLongitude(double longitude) { - this.longitude = longitude; - } - - public String getAddress() { - return address; - } - - public void setAddress(String address) { - this.address = address; - } - - @Override - public String toString() { - return "Station{" + - "id=" + id + - ", countryCode='" + countryCode + '\'' + - ", provider='" + provider + '\'' + - ", lac='" + lac + '\'' + - ", cell='" + cell + '\'' + - ", latitude=" + latitude + - ", longitude=" + longitude + - ", address='" + address + '\'' + - '}'; - } -} diff --git a/WebGisDemo/src/main/java/com/luxx/web/model/request/DataInCircleRequest.java b/WebGisDemo/src/main/java/com/luxx/web/model/request/DataInCircleRequest.java deleted file mode 100644 index a288d37..0000000 --- a/WebGisDemo/src/main/java/com/luxx/web/model/request/DataInCircleRequest.java +++ /dev/null @@ -1,40 +0,0 @@ -package com.luxx.web.model.request; - -public class DataInCircleRequest { - private double radius; - private double lat; - private double lng; - - public double getRadius() { - return radius; - } - - public void setRadius(double radius) { - this.radius = radius; - } - - public double getLat() { - return lat; - } - - public void setLat(double lat) { - this.lat = lat; - } - - public double getLng() { - return lng; - } - - public void setLng(double lng) { - this.lng = lng; - } - - @Override - public String toString() { - return "DataInCircleRequest{" + - "radius=" + radius + - ", lat=" + lat + - ", lng=" + lng + - '}'; - } -} diff --git a/WebGisDemo/src/main/java/com/luxx/web/model/request/DataInRectangleRequest.java b/WebGisDemo/src/main/java/com/luxx/web/model/request/DataInRectangleRequest.java deleted file mode 100644 index 90b7fa5..0000000 --- a/WebGisDemo/src/main/java/com/luxx/web/model/request/DataInRectangleRequest.java +++ /dev/null @@ -1,24 +0,0 @@ -package com.luxx.web.model.request; - -import com.luxx.web.model.PoiPoint; - -import java.util.List; - -public class DataInRectangleRequest { - private List points; - - public List getPoints() { - return points; - } - - public void setPoints(List points) { - this.points = points; - } - - @Override - public String toString() { - return "DataInRectangleRequest{" + - "points=" + points + - '}'; - } -} diff --git a/WebGisDemo/src/main/resources/application-dev.properties b/WebGisDemo/src/main/resources/application-dev.properties deleted file mode 100644 index 10e3e84..0000000 --- a/WebGisDemo/src/main/resources/application-dev.properties +++ /dev/null @@ -1,7 +0,0 @@ -logging.config=classpath:log4j/log4j2-dev.xml -# spring datasource -spring.datasource.driver-class-name=com.mysql.cj.jdbc.Driver -spring.datasource.url=jdbc:mysql://127.0.0.1:3306/web?useAffectedRows=true&useLegacyDatetimeCode=false&serverTimezone=Asia/Shanghai -spring.datasource.username=123 -spring.datasource.password=123 -spring.datasource.type=com.zaxxer.hikari.HikariDataSource \ No newline at end of file diff --git a/WebGisDemo/src/main/resources/application-prd.properties b/WebGisDemo/src/main/resources/application-prd.properties deleted file mode 100644 index ed6ee3d..0000000 --- a/WebGisDemo/src/main/resources/application-prd.properties +++ /dev/null @@ -1,7 +0,0 @@ -logging.config=classpath:log4j/log4j2-prd.xml -# spring datasource -spring.datasource.driver-class-name=com.mysql.cj.jdbc.Driver -spring.datasource.url=jdbc:mysql://127.0.0.1:3306/web?useAffectedRows=true&useLegacyDatetimeCode=false&serverTimezone=Asia/Shanghai -spring.datasource.username=123 -spring.datasource.password=123 -spring.datasource.type=com.zaxxer.hikari.HikariDataSource \ No newline at end of file diff --git a/WebGisDemo/src/main/resources/application.properties b/WebGisDemo/src/main/resources/application.properties index db924aa..f8fc3e6 100644 --- a/WebGisDemo/src/main/resources/application.properties +++ b/WebGisDemo/src/main/resources/application.properties @@ -1,9 +1,9 @@ -spring.profiles.active=dev +#spring.profiles.active=dev server.port=9090 # resources spring.mvc.servlet.load-on-startup=0 spring.mvc.static-path-pattern=/static/** -#spring.resources.static-locations=classpath:/static/ +spring.mvc.servlet.path=/ # freemarker spring.freemarker.templateLoaderPath=classpath:/templates/ spring.freemarker.suffix=.ftl @@ -11,4 +11,17 @@ spring.freemarker.charset=UTF-8 spring.freemarker.request-context-attribute=request spring.freemarker.settings.number_format=0.########## # spring mybatis -mybatis.mapper-locations=classpath*:/mapper/*.xml \ No newline at end of file +mybatis.mapper-locations=classpath*:/mapper/*.xml +logging.config=classpath:log4j/log4j2-dev.xml +# spring datasource +spring.datasource.url=jdbc:mysql://127.0.0.1:3306/data_db?useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&allowPublicKeyRetrieval=true&useSSL=false&zeroDateTimeBehavior=convertToNull&serverTimezone=GMT%2B8 +spring.datasource.username=root +spring.datasource.password=admin +# ES +es.address=127.0.0.1:9200 +# POI index DB +index.db.url=jdbc:mysql://127.0.0.1:3306/data_db?useLegacyDatetimeCode=false&serverTimezone=Asia/Shanghai +index.db.username=root +index.db.password=admin +index.db.table=station +index.db.type=mysql diff --git a/WebGisDemo/src/main/resources/db_table/station_table.sql b/WebGisDemo/src/main/resources/db_table/station_table.sql deleted file mode 100644 index 87934b0..0000000 --- a/WebGisDemo/src/main/resources/db_table/station_table.sql +++ /dev/null @@ -1,27 +0,0 @@ -DROP TABLE IF EXISTS `station`; -CREATE TABLE `station` ( - `id` bigint(20) NOT NULL AUTO_INCREMENT, - `country_code` varchar(16) DEFAULT '', - `provider` varchar(32) DEFAULT '', - `lac` varchar(32) DEFAULT '', - `cell` varchar(32) DEFAULT '', - `latitude` DOUBLE(20,8) DEFAULT NULL, - `longitude` DOUBLE(20,8) DEFAULT NULL, - `address` varchar(512) DEFAULT '', - PRIMARY KEY (`id`) -) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='station table' - -INSERT INTO `dataservice`.`station` (`id`, `country_code`, `provider`, `lac`, `cell`, `latitude`, `longitude`, `address`) -VALUES ('1', '406', '0', '111', '222', '32.12800000', '118.77420000', '南京测试点1'); - -INSERT INTO `dataservice`.`station` (`id`, `country_code`, `provider`, `lac`, `cell`, `latitude`, `longitude`, `address`) -VALUES ('2', '406', '1', '123', '234', '32.22700000', '118.86420000', '南京测试点2'); - -INSERT INTO `dataservice`.`station` (`id`, `country_code`, `provider`, `lac`, `cell`, `latitude`, `longitude`, `address`) -VALUES ('3', '406', '0', '222', '333', '30.32600000', '118.78420000', '南京测试点3'); - -INSERT INTO `dataservice`.`station` (`id`, `country_code`, `provider`, `lac`, `cell`, `latitude`, `longitude`, `address`) -VALUES ('4', '406', '1', '123', '234', '31.22700000', '118.56420000', '南京测试点4'); - -INSERT INTO `dataservice`.`station` (`id`, `country_code`, `provider`, `lac`, `cell`, `latitude`, `longitude`, `address`) -VALUES ('5', '406', '0', '222', '333', '32.32600000', '118.68420000', '南京测试点5'); diff --git a/WebGisDemo/src/main/resources/log4j/log4j2-dev.xml b/WebGisDemo/src/main/resources/log4j/log4j2-dev.xml index 39475c2..d00b831 100644 --- a/WebGisDemo/src/main/resources/log4j/log4j2-dev.xml +++ b/WebGisDemo/src/main/resources/log4j/log4j2-dev.xml @@ -1,39 +1,45 @@ - - ${sys:user.dir}/logs/ - app - - - - - - - - - - - - - - + + ${sys:user.dir}/logs/ + app + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/WebGisDemo/src/main/resources/log4j/log4j2-prd.xml b/WebGisDemo/src/main/resources/log4j/log4j2-prd.xml index d06cce5..3fc5f39 100644 --- a/WebGisDemo/src/main/resources/log4j/log4j2-prd.xml +++ b/WebGisDemo/src/main/resources/log4j/log4j2-prd.xml @@ -6,25 +6,31 @@ - + filePattern="${LOG_HOME}/$${date:yyyy-MM}/${FILE_NAME}-%d{yyyy-MM-dd}-%i.log" + name="append-file-log"> + - - + + - + + + + + + - + - + - + \ No newline at end of file diff --git a/WebGisDemo/src/main/resources/mapper/StationMapper.xml b/WebGisDemo/src/main/resources/mapper/StationMapper.xml index a500645..6386faa 100644 --- a/WebGisDemo/src/main/resources/mapper/StationMapper.xml +++ b/WebGisDemo/src/main/resources/mapper/StationMapper.xml @@ -1,8 +1,8 @@ - - + + @@ -14,14 +14,14 @@ - id, - country_code, - provider, - lac, - cell, - latitude, - longitude, - address + id, + country_code, + provider, + lac, + cell, + latitude, + longitude, + address + parameterType="com.luxx.gis.model.Station"> INSERT INTO station @@ -86,7 +86,7 @@ WHERE id = #{id} - + UPDATE station country_code = #{countryCode}, diff --git a/pom.xml b/pom.xml index 2f0eb21..e731b7e 100644 --- a/pom.xml +++ b/pom.xml @@ -9,7 +9,6 @@ pom - IndexSearchService MapHttpService NettyMqService WebGisDemo @@ -20,7 +19,6 @@ UTF-8 1.8 1.8 - 2.2.7.RELEASE