work

src/interpreter/interpreter.cpp

@@ -263,6 +263,8 @@ struct ExpressionInterpreter : OverriddenVisitor<ExpressionInterpreter, Flow> {
   Flow visitArrayFill(ArrayFill* curr) { WASM_UNREACHABLE("TODO"); }
   Flow visitArrayInitData(ArrayInitData* curr) { WASM_UNREACHABLE("TODO"); }
   Flow visitArrayInitElem(ArrayInitElem* curr) { WASM_UNREACHABLE("TODO"); }
+  Flow visitArrayRMW(ArrayRMW* curr) { WASM_UNREACHABLE("TODO"); }
+  Flow visitArrayCmpxchg(ArrayCmpxchg* curr) { WASM_UNREACHABLE("TODO"); }
   Flow visitRefAs(RefAs* curr) { WASM_UNREACHABLE("TODO"); }
   Flow visitStringNew(StringNew* curr) { WASM_UNREACHABLE("TODO"); }
   Flow visitStringConst(StringConst* curr) { WASM_UNREACHABLE("TODO"); }

src/passes/OptimizeInstructions.cpp

@@ -2240,6 +2240,18 @@ struct OptimizeInstructions
     trapOnNull(curr, curr->destRef) || trapOnNull(curr, curr->srcRef);
   }
 
+  void visitArrayRMW(ArrayRMW* curr) {
+    skipNonNullCast(curr->ref, curr);
+    trapOnNull(curr, curr->ref);
+    // TODO: more opts like StructRMW
+  }
+
+  void visitArrayCmpxchg(ArrayCmpxchg* curr) {
+    skipNonNullCast(curr->ref, curr);
+    trapOnNull(curr, curr->ref);
+    // TODO: more opts like StructCmpxchg
+  }
+
   void visitRefCast(RefCast* curr) {
     // Note we must check the ref's type here and not our own, since we only
     // refinalize at the end, which means our type may not have been updated yet

src/passes/Print.cpp

@@ -2444,6 +2444,26 @@ struct PrintExpressionContents
     o << ' ';
     curr->segment.print(o);
   }
+  void visitArrayRMW(ArrayRMW* curr) {
+    prepareColor(o);
+    o << "array.atomic.rmw.";
+    printAtomicRMWOp(curr->op);
+    restoreNormalColor(o);
+    o << ' ';
+    printMemoryOrder(curr->order);
+    printMemoryOrder(curr->order);
+    auto heapType = curr->ref->type.getHeapType();
+    printHeapTypeName(heapType);
+  }
+  void visitArrayCmpxchg(ArrayCmpxchg* curr) {
+    prepareColor(o);
+    o << "array.atomic.rmw.cmpxchg ";
+    restoreNormalColor(o);
+    printMemoryOrder(curr->order);
+    printMemoryOrder(curr->order);
+    auto heapType = curr->ref->type.getHeapType();
+    printHeapTypeName(heapType);
+  }
   void visitRefAs(RefAs* curr) {
     switch (curr->op) {
       case RefAsNonNull:

src/wasm/wasm-binary.cpp

@@ -3784,6 +3784,28 @@ Result<> WasmBinaryReader::readInst() {
           auto type = getIndexedHeapType();
           return builder.makeArraySet(type, order);
         }
+
+#define ARRAY_RMW(op)                                                         \
+  case BinaryConsts::ArrayAtomicRMW##op: {                                    \
+    auto order = getMemoryOrder(true);                                         \
+    auto type = getIndexedHeapType();                                          \
+    auto field = getU32LEB();                                                  \
+    return builder.makeArrayRMW(RMW##op, type, field, order);                 \
+  }
+
+          ARRAY_RMW(Add)
+          ARRAY_RMW(Sub)
+          ARRAY_RMW(And)
+          ARRAY_RMW(Or)
+          ARRAY_RMW(Xor)
+          ARRAY_RMW(Xchg)
+
+        case BinaryConsts::ArrayAtomicRMWCmpxchg: {
+          auto order = getMemoryOrder(true);
+          auto type = getIndexedHeapType();
+          auto field = getU32LEB();
+          return builder.makeArrayCmpxchg(type, field, order);
+        }
       }
       return Err{"unknown atomic operation " + std::to_string(op)};
     }

src/wasm/wasm-ir-builder.cpp

@@ -667,6 +667,20 @@ struct IRBuilder::ChildPopper
     return popConstrainedChildren(children);
   }
 
+  Result<> visitArrayRMW(ArrayRMW* curr,
+                          std::optional<HeapType> ht = std::nullopt) {
+    std::vector<Child> children;
+    ConstraintCollector{builder, children}.visitArrayRMW(curr, ht);
+    return popConstrainedChildren(children);
+  }
+
+  Result<> visitArrayCmpxchg(ArrayCmpxchg* curr,
+                              std::optional<HeapType> ht = std::nullopt) {
+    std::vector<Child> children;
+    ConstraintCollector{builder, children}.visitArrayCmpxchg(curr, ht);
+    return popConstrainedChildren(children);
+  }
+
   Result<> visitStringEncode(StringEncode* curr,
                              std::optional<HeapType> ht = std::nullopt) {
     std::vector<Child> children;
@@ -2369,6 +2383,26 @@ Result<> IRBuilder::makeArrayInitElem(HeapType type, Name elem) {
   return Ok{};
 }
 
+Result<> IRBuilder::makeArrayRMW(AtomicRMWOp op,
+                                  HeapType type,
+                                  MemoryOrder order) {
+  ArrayRMW curr;
+  CHECK_ERR(ChildPopper{*this}.visitArrayRMW(&curr, type));
+  CHECK_ERR(validateTypeAnnotation(type, curr.ref));
+  push(builder.makeArrayRMW(op, curr.ref, curr.index, curr.value, order));
+  return Ok{};
+}
+
+Result<>
+IRBuilder::makeArrayCmpxchg(HeapType type, MemoryOrder order) {
+  ArrayCmpxchg curr;
+  CHECK_ERR(ChildPopper{*this}.visitArrayCmpxchg(&curr, type));
+  CHECK_ERR(validateTypeAnnotation(type, curr.ref));
+  push(builder.makeArrayCmpxchg(
+    field, curr.ref, curr.ref, curr.expected, curr.replacement, order));
+  return Ok{};
+}
+
 Result<> IRBuilder::makeRefAs(RefAsOp op) {
   RefAs curr;
   curr.op = op;

src/wasm/wasm-stack.cpp

@@ -2552,6 +2552,49 @@ void BinaryInstWriter::visitArrayInitElem(ArrayInitElem* curr) {
   o << U32LEB(parent.getElementSegmentIndex(curr->segment));
 }
 
+void BinaryInstWriter::visitArrayRMW(ArrayRMW* curr) {
+  if (curr->ref->type.isNull()) {
+    emitUnreachable();
+    return;
+  }
+  o << int8_t(BinaryConsts::AtomicPrefix);
+  switch (curr->op) {
+    case RMWAdd:
+      o << U32LEB(BinaryConsts::ArrayAtomicRMWAdd);
+      break;
+    case RMWSub:
+      o << U32LEB(BinaryConsts::ArrayAtomicRMWSub);
+      break;
+    case RMWAnd:
+      o << U32LEB(BinaryConsts::ArrayAtomicRMWAnd);
+      break;
+    case RMWOr:
+      o << U32LEB(BinaryConsts::ArrayAtomicRMWOr);
+      break;
+    case RMWXor:
+      o << U32LEB(BinaryConsts::ArrayAtomicRMWXor);
+      break;
+    case RMWXchg:
+      o << U32LEB(BinaryConsts::ArrayAtomicRMWXchg);
+      break;
+  }
+  parent.writeMemoryOrder(curr->order, /*isRMW=*/true);
+  parent.writeIndexedHeapType(curr->ref->type.getHeapType());
+  o << U32LEB(curr->index);
+}
+
+void BinaryInstWriter::visitArrayCmpxchg(ArrayCmpxchg* curr) {
+  if (curr->ref->type.isNull()) {
+    emitUnreachable();
+    return;
+  }
+  o << int8_t(BinaryConsts::AtomicPrefix)
+    << U32LEB(BinaryConsts::ArrayAtomicRMWCmpxchg);
+  parent.writeMemoryOrder(curr->order, /*isRMW=*/true);
+  parent.writeIndexedHeapType(curr->ref->type.getHeapType());
+  o << U32LEB(curr->index);
+}
+
 void BinaryInstWriter::visitRefAs(RefAs* curr) {
   switch (curr->op) {
     case RefAsNonNull:

src/wasm/wasm-validator.cpp

@@ -515,6 +515,8 @@ struct FunctionValidator : public WalkerPass<PostWalker<FunctionValidator>> {
   template<typename ArrayInit> void visitArrayInit(ArrayInit* curr);
   void visitArrayInitData(ArrayInitData* curr);
   void visitArrayInitElem(ArrayInitElem* curr);
+  void visitArrayRMW(ArrayRMW* curr);
+  void visitArrayCmpxchg(ArrayCmpxchg* curr);
   void visitStringNew(StringNew* curr);
   void visitStringConst(StringConst* curr);
   void visitStringMeasure(StringMeasure* curr);
@@ -3692,6 +3694,108 @@ void FunctionValidator::visitArrayInitElem(ArrayInitElem* curr) {
                   "array.init_elem segment type must match destination type");
 }
 
+void FunctionValidator::visitArrayRMW(ArrayRMW* curr) {
+  auto expected =
+    FeatureSet::GC | FeatureSet::Atomics | FeatureSet::SharedEverything;
+  if (!shouldBeTrue(expected <= getModule()->features,
+                    curr,
+                    "array.atomic.rmw requires additional features ")) {
+    getStream() << getMissingFeaturesList(*getModule(), expected) << '\n';
+  }
+  if (curr->ref->type == Type::unreachable) {
+    return;
+  }
+  if (!shouldBeTrue(curr->ref->type.isRef(),
+                    curr->ref,
+                    "array.atomic.rmw ref must be a reference type")) {
+    return;
+  }
+  auto type = curr->ref->type.getHeapType();
+  if (type.isMaybeShared(HeapType::none)) {
+    return;
+  }
+  if (!shouldBeTrue(
+        type.isArray(), curr->ref, "array.atomic.rmw ref must be a array")) {
+    return;
+  }
+  auto field = GCTypeUtils::getField(curr->ref->type);
+  shouldBeEqual(
+    field.mutable_, Mutable, curr, "array.atomic.rmw field must be mutable");
+  shouldBeFalse(
+    field.isPacked(), curr, "array.atomic.rmw field must not be packed");
+  bool isAny =
+    field.type.isRef() &&
+    Type::isSubType(
+      field.type,
+      Type(HeapTypes::any.getBasic(field.type.getHeapType().getShared()),
+           Nullable));
+  if (!shouldBeTrue(field.type == Type::i32 || field.type == Type::i64 ||
+                      (isAny && curr->op == RMWXchg),
+                    curr,
+                    "array.atomic.rmw field type invalid for operation")) {
+    return;
+  }
+  shouldBeSubType(curr->value->type,
+                  field.type,
+                  curr,
+                  "array.atomic.rmw value must have the proper type");
+}
+
+void FunctionValidator::visitArrayCmpxchg(ArrayCmpxchg* curr) {
+  auto expected =
+    FeatureSet::GC | FeatureSet::Atomics | FeatureSet::SharedEverything;
+  if (!shouldBeTrue(expected <= getModule()->features,
+                    curr,
+                    "array.atomic.rmw requires additional features ")) {
+    getStream() << getMissingFeaturesList(*getModule(), expected) << '\n';
+  }
+  if (curr->ref->type == Type::unreachable) {
+    return;
+  }
+  if (!shouldBeTrue(curr->ref->type.isRef(),
+                    curr->ref,
+                    "array.atomic.rmw ref must be a reference type")) {
+    return;
+  }
+  auto type = curr->ref->type.getHeapType();
+  if (type.isMaybeShared(HeapType::none)) {
+    return;
+  }
+  if (!shouldBeTrue(
+        type.isArray(), curr->ref, "array.atomic.rmw ref must be a array")) {
+    return;
+  }
+  auto field = GCTypeUtils::getField(curr->ref->type);
+  shouldBeEqual(
+    field.mutable_, Mutable, curr, "array.atomic.rmw field must be mutable");
+  shouldBeFalse(
+    field.isPacked(), curr, "array.atomic.rmw field must not be packed");
+
+  Type expectedExpectedType;
+  if (field.type == Type::i32) {
+    expectedExpectedType = Type::i32;
+  } else if (field.type == Type::i64) {
+    expectedExpectedType = Type::i64;
+  } else if (field.type.isRef()) {
+    expectedExpectedType = Type(
+      HeapTypes::eq.getBasic(field.type.getHeapType().getShared()), Nullable);
+  } else {
+    shouldBeTrue(
+      false, curr, "array.atomic.rmw field type invalid for operation");
+    return;
+  }
+  shouldBeSubType(
+    curr->expected->type,
+    expectedExpectedType,
+    curr,
+    "array.atomic.rmw.cmpxchg expected value must have the proper type");
+  shouldBeSubType(
+    curr->replacement->type,
+    field.type,
+    curr,
+    "array.atomic.rmw.cmpxchg replacement value must have the proper type");
+}
+
 void FunctionValidator::visitStringNew(StringNew* curr) {
   shouldBeTrue(!getModule() || getModule()->features.hasStrings(),
                curr,

src/wasm/wasm.cpp

@@ -1362,6 +1362,34 @@ void ArrayInitElem::finalize() {
   }
 }
 
+void ArrayRMW::finalize() {
+  if (ref->type == Type::unreachable || index->type == Type::Unreachable ||
+      value->type == Type::unreachable) {
+    type = Type::unreachable;
+  } else if (ref->type.isNull()) {
+    // We have no array type to read the field off of, but the most precise
+    // possible option is the type of the value we are using to make the
+    // modification.
+    type = value->type;
+  } else {
+    type = ref->type.getHeapType().getArray().element.type;
+  }
+}
+
+void ArrayCmpxchg::finalize() {
+  if (ref->type == Type::unreachable || index->type == Type::Unreachable ||
+      expected->type == Type::unreachable ||
+      replacement->type == Type::unreachable) {
+    type = Type::unreachable;
+  } else if (ref->type.isNull()) {
+    // Like ArrayRMW, but the most precise possible field type is the LUB of
+    // the expected and replacement values.
+    type = Type::getLeastUpperBound(expected->type, replacement->type);
+  } else {
+    type = ref->type.getHeapType().getArray().element.type;
+  }
+}
+
 void RefAs::finalize() {
   // An unreachable child means we are unreachable. Also set ourselves to
   // unreachable when the child is invalid (say, it is an i32 or some other non-