Skip to content
This repository was archived by the owner on Apr 22, 2020. It is now read-only.

Parallelise batching of writes for similarity algorithms #814

Open
wants to merge 11 commits into
base: 3.4
Choose a base branch
from
Open

Parallelise batching of writes for similarity algorithms #814

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
2777a82
batching wip
mneedham Feb 11, 2019
714545d
return write duration
mneedham Feb 11, 2019
e99384b
adding tests for similarity exporter
mneedham Feb 12, 2019
cd79fec
more tests
mneedham Feb 12, 2019
36ad67e
parameterise similarity test
mneedham Feb 12, 2019
f519abf
simplify the code for working out partitions
mneedham Feb 12, 2019
35cbc2b
remove duplication
mneedham Feb 12, 2019
c75886c
push more code into the base SimilarityExporter class
mneedham Feb 12, 2019
78ac242
remove spaces + tidy up imports
mneedham Feb 12, 2019
aed8535
package private some things
mneedham Feb 12, 2019
93691f7
writeMillis instead of writeDuration
mneedham Feb 12, 2019
File filter

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
writeMillis instead of writeDuration
  • Loading branch information
@mneedham
mneedham committed Feb 12, 2019
commit 93691f70509f8369cdb3810e957c9a14c4afb1f1
10 changes: 5 additions & 5 deletions algo/src/main/java/org/neo4j/graphalgo/similarity/SimilaritySummaryResult.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@

public class SimilaritySummaryResult {

public final long writeDuration;
public final long writeMillis;
public final long nodes;
public final long similarityPairs;
public final boolean write;
Expand All @@ -47,7 +47,7 @@ public SimilaritySummaryResult(long nodes, long similarityPairs,
boolean write, String writeRelationshipType, String writeProperty,
double min, double max, double mean, double stdDev,
double p25, double p50, double p75, double p90, double p95,
double p99, double p999, double p100, long writeDuration) {
double p99, double p999, double p100, long writeMillis) {
this.nodes = nodes;
this.similarityPairs = similarityPairs;
this.write = write;
Expand All @@ -65,10 +65,10 @@ public SimilaritySummaryResult(long nodes, long similarityPairs,
this.p99 = p99;
this.p999 = p999;
this.p100 = p100;
this.writeDuration = writeDuration;
this.writeMillis = writeMillis;
}

static SimilaritySummaryResult from(long length, AtomicLong similarityPairs, String writeRelationshipType, String writeProperty, boolean write, DoubleHistogram histogram, long writeDuration) {
static SimilaritySummaryResult from(long length, AtomicLong similarityPairs, String writeRelationshipType, String writeProperty, boolean write, DoubleHistogram histogram, long writeMillis) {
return new SimilaritySummaryResult(
length,
similarityPairs.get(),
Expand All @@ -87,7 +87,7 @@ static SimilaritySummaryResult from(long length, AtomicLong similarityPairs, Str
histogram.getValueAtPercentile(99D),
histogram.getValueAtPercentile(99.9D),
histogram.getValueAtPercentile(100D),
writeDuration
writeMillis
);
}
}
24 changes: 24 additions & 0 deletions tests/src/test/java/org/neo4j/graphalgo/similarity/SimilarityExporterTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,30 @@ public void smallerThanBatchSize() throws Throwable {
}
}

@Test
public void disconnectedUpdates() throws Throwable {
int nodeCount = 6;
createNodes(api, nodeCount);
exporter = load(similarityExporterFactory, nodeCount);

Stream<SimilarityResult> similarityPairs = Stream.of(
new SimilarityResult(0, 1, -1, -1, -1, 0.5),
new SimilarityResult(2, 3, -1, -1, -1, 0.7),
new SimilarityResult(4, 5, -1, -1, -1, 0.9)
);

exporter.export(similarityPairs, 10);

try (Transaction tx = api.beginTx()) {
List<SimilarityRelationship> allRelationships = getSimilarityRelationships(api);

assertThat(allRelationships, hasSize(3));
assertThat(allRelationships, hasItems(new SimilarityRelationship(0, 1, 0.5)));
assertThat(allRelationships, hasItems(new SimilarityRelationship(2, 3, 0.7)));
assertThat(allRelationships, hasItems(new SimilarityRelationship(4, 5, 0.9)));
}
}

private List<SimilarityRelationship> getSimilarityRelationships(GraphDatabaseAPI api) {
return api.getAllRelationships().stream()
.map(relationship -> new SimilarityRelationship(relationship.getStartNodeId(), relationship.getEndNodeId(), (double)relationship.getProperty(PROPERTY_NAME)))
Expand Down