METAMODEL-1165: Fixed - added default_table alias table
[metamodel.git] / neo4j / src / main / java / org / apache / metamodel / neo4j / Neo4jDataContext.java
1 /**
2 * Licensed to the Apache Software Foundation (ASF) under one
3 * or more contributor license agreements. See the NOTICE file
4 * distributed with this work for additional information
5 * regarding copyright ownership. The ASF licenses this file
6 * to you under the Apache License, Version 2.0 (the
7 * "License"); you may not use this file except in compliance
8 * with the License. You may obtain a copy of the License at
9 *
10 * http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing,
13 * software distributed under the License is distributed on an
14 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 * KIND, either express or implied. See the License for the
16 * specific language governing permissions and limitations
17 * under the License.
18 */
19 package org.apache.metamodel.neo4j;
20
21 import java.util.ArrayList;
22 import java.util.LinkedHashSet;
23 import java.util.List;
24 import java.util.Set;
25 import java.util.stream.Collectors;
26
27 import org.apache.http.HttpHost;
28 import org.apache.http.client.methods.HttpGet;
29 import org.apache.http.impl.client.CloseableHttpClient;
30 import org.apache.http.impl.client.HttpClientBuilder;
31 import org.apache.metamodel.DataContext;
32 import org.apache.metamodel.MetaModelException;
33 import org.apache.metamodel.QueryPostprocessDataContext;
34 import org.apache.metamodel.data.DataSet;
35 import org.apache.metamodel.data.DocumentSource;
36 import org.apache.metamodel.query.FilterItem;
37 import org.apache.metamodel.query.SelectItem;
38 import org.apache.metamodel.schema.Column;
39 import org.apache.metamodel.schema.MutableSchema;
40 import org.apache.metamodel.schema.MutableTable;
41 import org.apache.metamodel.schema.Schema;
42 import org.apache.metamodel.schema.Table;
43 import org.apache.metamodel.schema.builder.DocumentSourceProvider;
44 import org.apache.metamodel.util.SimpleTableDef;
45 import org.json.JSONArray;
46 import org.json.JSONException;
47 import org.json.JSONObject;
48 import org.slf4j.Logger;
49 import org.slf4j.LoggerFactory;
50
51 /**
52 * DataContext implementation for Neo4j
53 */
54 public class Neo4jDataContext extends QueryPostprocessDataContext implements DataContext, DocumentSourceProvider {
55
56 public static final Logger logger = LoggerFactory.getLogger(Neo4jDataContext.class);
57
58 public static final String SCHEMA_NAME = "neo4j";
59
60 public static final int DEFAULT_PORT = 7474;
61
62 public static final String RELATIONSHIP_PREFIX = "rel_";
63
64 public static final String RELATIONSHIP_COLUMN_SEPARATOR = "#";
65
66 private final SimpleTableDef[] _tableDefs;
67
68 private final Neo4jRequestWrapper _requestWrapper;
69
70 private final HttpHost _httpHost;
71
72 private String _serviceRoot = "/db/data";
73
74 public Neo4jDataContext(String hostname, int port, String username, String password, SimpleTableDef... tableDefs) {
75 super(false);
76 _httpHost = new HttpHost(hostname, port);
77 final CloseableHttpClient httpClient = HttpClientBuilder.create().build();
78 _requestWrapper = new Neo4jRequestWrapper(httpClient, _httpHost, username, password, _serviceRoot);
79 _tableDefs = tableDefs;
80 }
81
82 public Neo4jDataContext(String hostname, int port, String username, String password, String serviceRoot,
83 SimpleTableDef... tableDefs) {
84 super(false);
85 _httpHost = new HttpHost(hostname, port);
86 final CloseableHttpClient httpClient = HttpClientBuilder.create().build();
87 _requestWrapper = new Neo4jRequestWrapper(httpClient, _httpHost, username, password, _serviceRoot);
88 _tableDefs = tableDefs;
89 _serviceRoot = serviceRoot;
90 }
91
92 public Neo4jDataContext(String hostname, int port, String username, String password) {
93 super(false);
94 _httpHost = new HttpHost(hostname, port);
95 final CloseableHttpClient httpClient = HttpClientBuilder.create().build();
96 _requestWrapper = new Neo4jRequestWrapper(httpClient, _httpHost, username, password, _serviceRoot);
97 _tableDefs = detectTableDefs();
98 }
99
100 public Neo4jDataContext(String hostname, int port, String username, String password, String serviceRoot) {
101 super(false);
102 _httpHost = new HttpHost(hostname, port);
103 final CloseableHttpClient httpClient = HttpClientBuilder.create().build();
104 _requestWrapper = new Neo4jRequestWrapper(httpClient, _httpHost, username, password, _serviceRoot);
105 _tableDefs = detectTableDefs();
106 _serviceRoot = serviceRoot;
107 }
108
109 public Neo4jDataContext(String hostname, int port, CloseableHttpClient httpClient) {
110 super(false);
111 _httpHost = new HttpHost(hostname, port);
112 _requestWrapper = new Neo4jRequestWrapper(httpClient, _httpHost, _serviceRoot);
113 _tableDefs = detectTableDefs();
114 }
115
116 public Neo4jDataContext(String hostname, int port, CloseableHttpClient httpClient, String serviceRoot) {
117 super(false);
118 _httpHost = new HttpHost(hostname, port);
119 _requestWrapper = new Neo4jRequestWrapper(httpClient, _httpHost, _serviceRoot);
120 _tableDefs = detectTableDefs();
121 _serviceRoot = serviceRoot;
122 }
123
124 public Neo4jDataContext(String hostname, int port, CloseableHttpClient httpClient, SimpleTableDef... tableDefs) {
125 super(false);
126 _httpHost = new HttpHost(hostname, port);
127 _requestWrapper = new Neo4jRequestWrapper(httpClient, _httpHost, _serviceRoot);
128 _tableDefs = tableDefs;
129 }
130
131 public Neo4jDataContext(String hostname, int port, CloseableHttpClient httpClient, String serviceRoot,
132 SimpleTableDef... tableDefs) {
133 super(false);
134 _httpHost = new HttpHost(hostname, port);
135 _requestWrapper = new Neo4jRequestWrapper(httpClient, _httpHost, _serviceRoot);
136 _tableDefs = tableDefs;
137 _serviceRoot = serviceRoot;
138 }
139
140 @Override
141 protected String getDefaultSchemaName() throws MetaModelException {
142 return SCHEMA_NAME;
143 }
144
145 @Override
146 protected Schema getMainSchema() throws MetaModelException {
147 MutableSchema schema = new MutableSchema(getMainSchemaName());
148 for (SimpleTableDef tableDef : _tableDefs) {
149 MutableTable table = tableDef.toTable().setSchema(schema);
150 schema.addTable(table);
151 }
152 return schema;
153 }
154
155 @Override
156 protected String getMainSchemaName() throws MetaModelException {
157 return SCHEMA_NAME;
158 }
159
160 public SimpleTableDef[] detectTableDefs() {
161 List<SimpleTableDef> tableDefs = new ArrayList<SimpleTableDef>();
162
163 String labelsJsonString = _requestWrapper.executeRestRequest(new HttpGet(_serviceRoot + "/labels"));
164
165 JSONArray labelsJsonArray;
166 try {
167 labelsJsonArray = new JSONArray(labelsJsonString);
168 for (int i = 0; i < labelsJsonArray.length(); i++) {
169 String label = labelsJsonArray.getString(i);
170
171 List<JSONObject> nodesPerLabel = getAllNodesPerLabel(label);
172
173 List<String> propertiesPerLabel = new ArrayList<String>();
174 for (JSONObject node : nodesPerLabel) {
175 List<String> propertiesPerNode = getAllPropertiesPerNode(node);
176 for (String property : propertiesPerNode) {
177 if (!propertiesPerLabel.contains(property)) {
178 propertiesPerLabel.add(property);
179 }
180 }
181 }
182
183 Set<String> relationshipPropertiesPerLabel = new LinkedHashSet<String>();
184 for (JSONObject node : nodesPerLabel) {
185 Integer nodeId = (Integer) node.getJSONObject("metadata").get("id");
186 List<JSONObject> relationshipsPerNode = getOutgoingRelationshipsPerNode(nodeId);
187 for (JSONObject relationship : relationshipsPerNode) {
188 // Add the relationship as a column in the table
189 String relationshipName = relationship.getString("type");
190 String relationshipNameProperty = RELATIONSHIP_PREFIX + relationshipName;
191 if (!relationshipPropertiesPerLabel.contains(relationshipNameProperty)) {
192 relationshipPropertiesPerLabel.add(relationshipNameProperty);
193 }
194
195 // Add all the relationship properties as table columns
196 List<String> propertiesPerRelationship = getAllPropertiesPerRelationship(relationship);
197 relationshipPropertiesPerLabel.addAll(propertiesPerRelationship);
198 }
199 }
200 propertiesPerLabel.addAll(relationshipPropertiesPerLabel);
201
202 // Do not add a table if label has no nodes (empty tables are
203 // considered non-existent)
204 if (!nodesPerLabel.isEmpty()) {
205 SimpleTableDef tableDef = new SimpleTableDef(label,
206 propertiesPerLabel.toArray(new String[propertiesPerLabel.size()]));
207 tableDefs.add(tableDef);
208 }
209 }
210 return tableDefs.toArray(new SimpleTableDef[tableDefs.size()]);
211 } catch (JSONException e) {
212 logger.error("Error occured in parsing JSON while detecting the schema: ", e);
213 throw new IllegalStateException(e);
214 }
215 }
216
217 private List<String> getAllPropertiesPerRelationship(JSONObject relationship) {
218 List<String> propertyNames = new ArrayList<String>();
219 try {
220 String relationshipName = RELATIONSHIP_PREFIX + relationship.getJSONObject("metadata").getString("type");
221 JSONObject relationshipPropertiesJSONObject = relationship.getJSONObject("data");
222 if (relationshipPropertiesJSONObject.length() > 0) {
223 JSONArray relationshipPropertiesNamesJSONArray = relationshipPropertiesJSONObject.names();
224 for (int i = 0; i < relationshipPropertiesNamesJSONArray.length(); i++) {
225 String propertyName = relationshipName + RELATIONSHIP_COLUMN_SEPARATOR
226 + relationshipPropertiesNamesJSONArray.getString(i);
227 if (!propertyNames.contains(propertyName)) {
228 propertyNames.add(propertyName);
229 }
230 }
231 }
232 return propertyNames;
233 } catch (JSONException e) {
234 logger.error("Error occured in parsing JSON while getting relationship properties: ", e);
235 throw new IllegalStateException(e);
236 }
237 }
238
239 private List<JSONObject> getOutgoingRelationshipsPerNode(Integer nodeId) {
240 List<JSONObject> outgoingRelationshipsPerNode = new ArrayList<JSONObject>();
241
242 String outgoingRelationshipsPerNodeJsonString = _requestWrapper.executeRestRequest(new HttpGet(_serviceRoot
243 + "/node/" + nodeId + "/relationships/out"));
244
245 JSONArray outgoingRelationshipsPerNodeJsonArray;
246 try {
247 outgoingRelationshipsPerNodeJsonArray = new JSONArray(outgoingRelationshipsPerNodeJsonString);
248 for (int i = 0; i < outgoingRelationshipsPerNodeJsonArray.length(); i++) {
249 JSONObject relationship = outgoingRelationshipsPerNodeJsonArray.getJSONObject(i);
250 if (!outgoingRelationshipsPerNode.contains(relationship)) {
251 outgoingRelationshipsPerNode.add(relationship);
252 }
253 }
254 return outgoingRelationshipsPerNode;
255 } catch (JSONException e) {
256 logger.error("Error occured in parsing JSON while detecting outgoing relationships for node: " + nodeId, e);
257 throw new IllegalStateException(e);
258 }
259 }
260
261 private List<JSONObject> getAllNodesPerLabel(String label) {
262 List<JSONObject> allNodesPerLabel = new ArrayList<JSONObject>();
263
264 String allNodesForLabelJsonString = _requestWrapper.executeRestRequest(new HttpGet(_serviceRoot + "/label/"
265 + label + "/nodes"));
266
267 JSONArray allNodesForLabelJsonArray;
268 try {
269 allNodesForLabelJsonArray = new JSONArray(allNodesForLabelJsonString);
270 for (int i = 0; i < allNodesForLabelJsonArray.length(); i++) {
271 JSONObject node = allNodesForLabelJsonArray.getJSONObject(i);
272 allNodesPerLabel.add(node);
273 }
274 return allNodesPerLabel;
275 } catch (JSONException e) {
276 logger.error("Error occured in parsing JSON while detecting the nodes for a label: " + label, e);
277 throw new IllegalStateException(e);
278 }
279 }
280
281 private List<String> getAllPropertiesPerNode(JSONObject node) {
282 List<String> properties = new ArrayList<String>();
283 properties.add("_id");
284
285 String propertiesEndpoint;
286 try {
287 propertiesEndpoint = node.getString("properties");
288
289 String allPropertiesPerNodeJsonString = _requestWrapper.executeRestRequest(new HttpGet(propertiesEndpoint));
290
291 JSONObject allPropertiesPerNodeJsonObject = new JSONObject(allPropertiesPerNodeJsonString);
292 for (int j = 0; j < allPropertiesPerNodeJsonObject.length(); j++) {
293 JSONArray propertiesJsonArray = allPropertiesPerNodeJsonObject.names();
294 for (int k = 0; k < propertiesJsonArray.length(); k++) {
295 String property = propertiesJsonArray.getString(k);
296 properties.add(property);
297 }
298 }
299 return properties;
300 } catch (JSONException e) {
301 logger.error("Error occured in parsing JSON while detecting the properties of a node: " + node, e);
302 throw new IllegalStateException(e);
303 }
304 }
305
306 @Override
307 protected DataSet materializeMainSchemaTable(Table table, List<Column> columns, int firstRow, int maxRows) {
308 if ((columns != null) && (columns.size() > 0)) {
309 Neo4jDataSet dataSet = null;
310 try {
311 String selectQuery = Neo4jCypherQueryBuilder.buildSelectQuery(table, columns, firstRow, maxRows);
312 String responseJSONString = _requestWrapper.executeCypherQuery(selectQuery);
313 JSONObject resultJSONObject = new JSONObject(responseJSONString);
314 final List<SelectItem> selectItems = columns.stream().map(SelectItem::new).collect(Collectors.toList());
315 dataSet = new Neo4jDataSet(selectItems, resultJSONObject);
316 } catch (JSONException e) {
317 logger.error("Error occured in parsing JSON while materializing the schema: ", e);
318 throw new IllegalStateException(e);
319 }
320
321 return dataSet;
322 } else {
323 logger.error("Encountered null or empty columns array for materializing main schema table.");
324 throw new IllegalArgumentException("Columns cannot be null or empty array");
325 }
326 }
327
328 @Override
329 protected DataSet materializeMainSchemaTable(Table table, List<Column> columns, int maxRows) {
330 return materializeMainSchemaTable(table, columns, 1, maxRows);
331 }
332
333 @Override
334 protected Number executeCountQuery(Table table, List<FilterItem> whereItems, boolean functionApproximationAllowed) {
335 String countQuery = Neo4jCypherQueryBuilder.buildCountQuery(table.getName(), whereItems);
336 String jsonResponse = _requestWrapper.executeCypherQuery(countQuery);
337
338 JSONObject jsonResponseObject;
339 try {
340 jsonResponseObject = new JSONObject(jsonResponse);
341 JSONArray resultsJSONArray = jsonResponseObject.getJSONArray("results");
342 JSONObject resultJSONObject = (JSONObject) resultsJSONArray.get(0);
343 JSONArray dataJSONArray = resultJSONObject.getJSONArray("data");
344 JSONObject rowJSONObject = (JSONObject) dataJSONArray.get(0);
345 JSONArray valueJSONArray = rowJSONObject.getJSONArray("row");
346 Number value = (Number) valueJSONArray.get(0);
347 return value;
348 } catch (JSONException e) {
349 logger.error("Error occured in parsing JSON response: ", e);
350 // Do not throw an exception here. Returning null here will make
351 // MetaModel attempt to count records manually and therefore recover
352 // from the error.
353 return null;
354 }
355 }
356
357 @Override
358 public DocumentSource getMixedDocumentSourceForSampling() {
359 return null;
360 }
361
362 @Override
363 public DocumentSource getDocumentSourceForTable(String sourceCollectionName) {
364 return null;
365 }
366 }