Skip to content
This repository was archived by the owner on Apr 22, 2020. It is now read-only.

Parallelise batching of writes for similarity algorithms #814

Open
wants to merge 11 commits into
base: 3.4
Choose a base branch
from
Prev Previous commit
Next Next commit
simplify the code for working out partitions
  • Loading branch information
mneedham committed Feb 12, 2019
commit f519abf69f12ddd97b9086e06825a8cf8df4cc67
4 changes: 0 additions & 4 deletions algo/src/main/java/org/neo4j/graphalgo/impl/DSSResult.java
Original file line number Diff line number Diff line change
Expand Up @@ -73,10 +73,6 @@ public Stream<DisjointSetStruct.Result> resultStream(IdMapping idMapping) {
: hugeStruct.resultStream(((HugeIdMapping) idMapping));
}

public Stream<DisjointSetStruct.InternalResult> internalResultStream(IdMapping idMapping) {
return struct.internalResultStream(idMapping);
}

public void forEach(NodeIterator nodes, IntIntPredicate consumer) {
if (struct != null) {
nodes.forEachNode(nodeId -> consumer.apply(nodeId, struct.find(nodeId)));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,6 @@
import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.LongAdder;
import java.util.stream.Collectors;
import java.util.stream.Stream;

Expand Down Expand Up @@ -82,7 +81,7 @@ public int export(Stream<SimilarityResult> similarityPairs, long batchSize) {
idMap.buildMappedIds();
HeavyGraph graph = new HeavyGraph(idMap, adjacencyMatrix, weightMap, Collections.emptyMap());

DSSResult dssResult = computePartitions(graph);
DisjointSetStruct dssResult = computePartitions(graph);

Stream<List<DisjointSetStruct.InternalResult>> stream = dssResult.internalResultStream(graph)
.collect(Collectors.groupingBy(item -> item.setId))
Expand Down Expand Up @@ -153,11 +152,11 @@ private static <T> void put(BlockingQueue<T> queue, T items) {
}
}

private DSSResult computePartitions(HeavyGraph graph) {
private DisjointSetStruct computePartitions(HeavyGraph graph) {
GraphUnionFind algo = new GraphUnionFind(graph);
DisjointSetStruct struct = algo.compute();
algo.release();
return new DSSResult(struct);
return struct;
}

private void export(SimilarityResult similarityResult) {
Expand Down