Bug 1923701 - Update vendored wasm2c to the latest version r=glandium

Differential Revision: https://phabricator.services.mozilla.com/D225090

config/external/wasm2c_sandbox_compiler/moz.build

@@ -25,7 +25,7 @@ HOST_SOURCES += [
     "/third_party/wasm2c/src/binary-reader-ir.cc",
     "/third_party/wasm2c/src/binary-reader-logging.cc",
     "/third_party/wasm2c/src/binary-reader-objdump.cc",
-    "/third_party/wasm2c/src/binary-reader-opcnt.cc",
+    "/third_party/wasm2c/src/binary-reader-stats.cc",
     "/third_party/wasm2c/src/binary-reader.cc",
     "/third_party/wasm2c/src/binary-writer-spec.cc",
     "/third_party/wasm2c/src/binary-writer.cc",
@@ -67,6 +67,7 @@ HOST_SOURCES += [
 # wasm2c sources
 HOST_SOURCES += [
     "/third_party/wasm2c/src/c-writer.cc",
+    "/third_party/wasm2c/src/prebuilt/wasm2c_atomicops_source_declarations.cc",
     "/third_party/wasm2c/src/prebuilt/wasm2c_header_bottom.cc",
     "/third_party/wasm2c/src/prebuilt/wasm2c_header_top.cc",
     "/third_party/wasm2c/src/prebuilt/wasm2c_simd_source_declarations.cc",

config/external/wasm2c_sandbox_compiler/moz.yaml

@@ -9,8 +9,8 @@ origin:
   description: wasm2c fork used for rlbox sandboxing
   url: https://github.com/WebAssembly/wabt
 
-  release: 963f973469b45969ce198e0c86d3af316790a780 (2023-05-12T21:56:46Z).
+  release: 94c25a93c5ee81939bd1acfee06808f6745883ce (2024-11-11T23:21:55Z).
-  revision: 963f973469b45969ce198e0c86d3af316790a780
+  revision: 94c25a93c5ee81939bd1acfee06808f6745883ce
 
   license: Apache-2.0
   license-file: LICENSE
@@ -31,6 +31,7 @@ vendoring:
     - src/template
     - test
     - third_party
+    - wasm2c/benchmarks
     - wasm2c/examples
     # files
     - .*
@@ -38,6 +39,7 @@ vendoring:
     - Contributing.md
     - Makefile
     - README.md
+    - SECURITY.md
     - ubsan.blacklist
     - src/tools/s*
     - src/tools/wasm-*

security/rlbox/rlbox.mozbuild

@@ -14,6 +14,7 @@ def RLBoxLibrary(name, use_segue=True):
 
     SOURCES += [f"!{name}.wasm.c"]
     SOURCES += ["/third_party/wasm2c/wasm2c/wasm-rt-impl.c"]
+    SOURCES += ["/third_party/wasm2c/wasm2c/wasm-rt-mem-impl.c"]
 
     # Configuration for the wasm2c runtime used by RLBox
 
@@ -31,7 +32,7 @@ def RLBoxLibrary(name, use_segue=True):
     DEFINES["WASM_RT_TRAP_HANDLER"] = "moz_wasm2c_trap_handler"
 
     # Don't limit the nested call depth
-    DEFINES["WASM_RT_USE_STACK_DEPTH_COUNT"] = 0
+    DEFINES["WASM_RT_NONCONFORMING_UNCHECKED_STACK_EXHAUSTION"] = 1
 
     # Configure the wasm runtime to invoke a callback when a Wasm memory growth
     # fails inside the sandbox. This information is used to annotate crash reports.

third_party/wasm2c/include/wabt/binary-reader-logging.h

@@ -74,7 +74,8 @@ class BinaryReaderLogging : public BinaryReaderDelegate {
                         std::string_view module_name,
                         std::string_view field_name,
                         Index memory_index,
-                        const Limits* page_limits) override;
+                        const Limits* page_limits,
+                        uint32_t page_size) override;
   Result OnImportGlobal(Index import_index,
                         std::string_view module_name,
                         std::string_view field_name,
@@ -102,7 +103,9 @@ class BinaryReaderLogging : public BinaryReaderDelegate {
 
   Result BeginMemorySection(Offset size) override;
   Result OnMemoryCount(Index count) override;
-  Result OnMemory(Index index, const Limits* limits) override;
+  Result OnMemory(Index index,
+                  const Limits* limits,
+                  uint32_t page_size) override;
   Result EndMemorySection() override;
 
   Result BeginGlobalSection(Offset size) override;
@@ -130,6 +133,7 @@ class BinaryReaderLogging : public BinaryReaderDelegate {
   Result BeginFunctionBody(Index index, Offset size) override;
   Result OnLocalDeclCount(Index count) override;
   Result OnLocalDecl(Index decl_index, Index count, Type type) override;
+  Result EndLocalDecls() override;
 
   Result OnOpcode(Opcode opcode) override;
   Result OnOpcodeBare() override;
@@ -200,7 +204,7 @@ class BinaryReaderLogging : public BinaryReaderDelegate {
   Result OnLocalSetExpr(Index local_index) override;
   Result OnLocalTeeExpr(Index local_index) override;
   Result OnLoopExpr(Type sig_type) override;
-  Result OnMemoryCopyExpr(Index srcmemidx, Index destmemidx) override;
+  Result OnMemoryCopyExpr(Index destmemidx, Index srcmemidx) override;
   Result OnDataDropExpr(Index segment_index) override;
   Result OnMemoryFillExpr(Index memidx) override;
   Result OnMemoryGrowExpr(Index memidx) override;
@@ -273,9 +277,8 @@ class BinaryReaderLogging : public BinaryReaderDelegate {
   Result EndElemSegmentInitExpr(Index index) override;
   Result OnElemSegmentElemType(Index index, Type elem_type) override;
   Result OnElemSegmentElemExprCount(Index index, Index count) override;
-  Result OnElemSegmentElemExpr_RefNull(Index segment_index, Type type) override;
+  Result BeginElemExpr(Index elem_index, Index expr_index) override;
-  Result OnElemSegmentElemExpr_RefFunc(Index segment_index,
+  Result EndElemExpr(Index elem_index, Index expr_index) override;
-                                       Index func_index) override;
   Result EndElemSegment(Index index) override;
   Result EndElemSection() override;
 
@@ -347,6 +350,12 @@ class BinaryReaderLogging : public BinaryReaderDelegate {
   Result OnDylinkExport(std::string_view name, uint32_t flags) override;
   Result EndDylinkSection() override;
 
+  Result BeginGenericCustomSection(Offset size) override;
+  Result OnGenericCustomSection(std::string_view name,
+                                const void* data,
+                                Offset size) override;
+  Result EndGenericCustomSection() override;
+
   Result BeginTargetFeaturesSection(Offset size) override;
   Result OnFeatureCount(Index count) override;
   Result OnFeature(uint8_t prefix, std::string_view name) override;

third_party/wasm2c/include/wabt/binary-reader-nop.h

@@ -89,7 +89,8 @@ class BinaryReaderNop : public BinaryReaderDelegate {
                         std::string_view module_name,
                         std::string_view field_name,
                         Index memory_index,
-                        const Limits* page_limits) override {
+                        const Limits* page_limits,
+                        uint32_t page_size) override {
     return Result::Ok;
   }
   Result OnImportGlobal(Index import_index,
@@ -130,7 +131,9 @@ class BinaryReaderNop : public BinaryReaderDelegate {
   /* Memory section */
   Result BeginMemorySection(Offset size) override { return Result::Ok; }
   Result OnMemoryCount(Index count) override { return Result::Ok; }
-  Result OnMemory(Index index, const Limits* limits) override {
+  Result OnMemory(Index index,
+                  const Limits* limits,
+                  uint32_t page_size) override {
     return Result::Ok;
   }
   Result EndMemorySection() override { return Result::Ok; }
@@ -172,6 +175,7 @@ class BinaryReaderNop : public BinaryReaderDelegate {
   Result OnLocalDecl(Index decl_index, Index count, Type type) override {
     return Result::Ok;
   }
+  Result EndLocalDecls() override { return Result::Ok; }
 
   /* Function expressions; called between BeginFunctionBody and
    EndFunctionBody */
@@ -273,7 +277,7 @@ class BinaryReaderNop : public BinaryReaderDelegate {
   Result OnLocalSetExpr(Index local_index) override { return Result::Ok; }
   Result OnLocalTeeExpr(Index local_index) override { return Result::Ok; }
   Result OnLoopExpr(Type sig_type) override { return Result::Ok; }
-  Result OnMemoryCopyExpr(Index srcmemidx, Index destmemidx) override {
+  Result OnMemoryCopyExpr(Index destmemidx, Index srcmemidx) override {
     return Result::Ok;
   }
   Result OnDataDropExpr(Index segment_index) override { return Result::Ok; }
@@ -370,12 +374,10 @@ class BinaryReaderNop : public BinaryReaderDelegate {
   Result OnElemSegmentElemExprCount(Index index, Index count) override {
     return Result::Ok;
   }
-  Result OnElemSegmentElemExpr_RefNull(Index segment_index,
+  Result BeginElemExpr(Index elem_index, Index expr_index) override {
-                                       Type type) override {
     return Result::Ok;
   }
-  Result OnElemSegmentElemExpr_RefFunc(Index segment_index,
+  Result EndElemExpr(Index elem_index, Index expr_index) override {
-                                       Index func_index) override {
     return Result::Ok;
   }
   Result EndElemSegment(Index index) override { return Result::Ok; }
@@ -521,6 +523,15 @@ class BinaryReaderNop : public BinaryReaderDelegate {
   }
   Result EndTargetFeaturesSection() override { return Result::Ok; }
 
+  /* Generic custom section */
+  Result BeginGenericCustomSection(Offset size) override { return Result::Ok; }
+  Result OnGenericCustomSection(std::string_view name,
+                                const void* data,
+                                Offset size) override {
+    return Result::Ok;
+  };
+  Result EndGenericCustomSection() override { return Result::Ok; }
+
   /* Linking section */
   Result BeginLinkingSection(Offset size) override { return Result::Ok; }
   Result OnSymbolCount(Index count) override { return Result::Ok; }

third_party/wasm2c/include/wabt/binary-reader-objdump.h

@@ -86,6 +86,7 @@ struct ObjdumpState {
   ObjdumpLocalNames local_names;
   std::vector<ObjdumpSymbol> symtab;
   std::map<Index, Index> function_param_counts;
+  std::map<Index, Index> function_types;
 };
 
 Result ReadBinaryObjdump(const uint8_t* data,

third_party/wasm2c/include/wabt/binary-reader.h

@@ -125,7 +125,8 @@ class BinaryReaderDelegate {
                                 std::string_view module_name,
                                 std::string_view field_name,
                                 Index memory_index,
-                                const Limits* page_limits) = 0;
+                                const Limits* page_limits,
+                                uint32_t page_size) = 0;
   virtual Result OnImportGlobal(Index import_index,
                                 std::string_view module_name,
                                 std::string_view field_name,
@@ -156,7 +157,9 @@ class BinaryReaderDelegate {
   /* Memory section */
   virtual Result BeginMemorySection(Offset size) = 0;
   virtual Result OnMemoryCount(Index count) = 0;
-  virtual Result OnMemory(Index index, const Limits* limits) = 0;
+  virtual Result OnMemory(Index index,
+                          const Limits* limits,
+                          uint32_t page_size) = 0;
   virtual Result EndMemorySection() = 0;
 
   /* Global section */
@@ -188,6 +191,7 @@ class BinaryReaderDelegate {
   virtual Result BeginFunctionBody(Index index, Offset size) = 0;
   virtual Result OnLocalDeclCount(Index count) = 0;
   virtual Result OnLocalDecl(Index decl_index, Index count, Type type) = 0;
+  virtual Result EndLocalDecls() = 0;
 
   /* Function expressions; called between BeginFunctionBody and
    EndFunctionBody */
@@ -269,7 +273,7 @@ class BinaryReaderDelegate {
   virtual Result OnLocalSetExpr(Index local_index) = 0;
   virtual Result OnLocalTeeExpr(Index local_index) = 0;
   virtual Result OnLoopExpr(Type sig_type) = 0;
-  virtual Result OnMemoryCopyExpr(Index srcmemidx, Index destmemidx) = 0;
+  virtual Result OnMemoryCopyExpr(Index destmemidx, Index srcmemidx) = 0;
   virtual Result OnDataDropExpr(Index segment_index) = 0;
   virtual Result OnMemoryFillExpr(Index memidx) = 0;
   virtual Result OnMemoryGrowExpr(Index memidx) = 0;
@@ -339,10 +343,8 @@ class BinaryReaderDelegate {
   virtual Result EndElemSegmentInitExpr(Index index) = 0;
   virtual Result OnElemSegmentElemType(Index index, Type elem_type) = 0;
   virtual Result OnElemSegmentElemExprCount(Index index, Index count) = 0;
-  virtual Result OnElemSegmentElemExpr_RefNull(Index segment_index,
+  virtual Result BeginElemExpr(Index elem_index, Index expr_index) = 0;
-                                               Type type) = 0;
+  virtual Result EndElemExpr(Index elem_index, Index expr_index) = 0;
-  virtual Result OnElemSegmentElemExpr_RefFunc(Index segment_index,
-                                               Index func_index) = 0;
   virtual Result EndElemSegment(Index index) = 0;
   virtual Result EndElemSection() = 0;
 
@@ -426,6 +428,13 @@ class BinaryReaderDelegate {
   virtual Result OnFeature(uint8_t prefix, std::string_view name) = 0;
   virtual Result EndTargetFeaturesSection() = 0;
 
+  /* Generic custom section */
+  virtual Result BeginGenericCustomSection(Offset size) = 0;
+  virtual Result OnGenericCustomSection(std::string_view name,
+                                        const void* data,
+                                        Offset size) = 0;
+  virtual Result EndGenericCustomSection() = 0;
+
   /* Linking section */
   virtual Result BeginLinkingSection(Offset size) = 0;
   virtual Result OnSymbolCount(Index count) = 0;

third_party/wasm2c/include/wabt/binary.h

@@ -24,7 +24,12 @@
 #define WABT_BINARY_LIMITS_HAS_MAX_FLAG 0x1
 #define WABT_BINARY_LIMITS_IS_SHARED_FLAG 0x2
 #define WABT_BINARY_LIMITS_IS_64_FLAG 0x4
-#define WABT_BINARY_LIMITS_ALL_FLAGS                                     \
+#define WABT_BINARY_LIMITS_HAS_CUSTOM_PAGE_SIZE_FLAG 0x8
+#define WABT_BINARY_LIMITS_ALL_MEMORY_FLAGS                              \
+  (WABT_BINARY_LIMITS_HAS_MAX_FLAG | WABT_BINARY_LIMITS_IS_SHARED_FLAG | \
+   WABT_BINARY_LIMITS_IS_64_FLAG |                                       \
+   WABT_BINARY_LIMITS_HAS_CUSTOM_PAGE_SIZE_FLAG)
+#define WABT_BINARY_LIMITS_ALL_TABLE_FLAGS                               \
   (WABT_BINARY_LIMITS_HAS_MAX_FLAG | WABT_BINARY_LIMITS_IS_SHARED_FLAG | \
    WABT_BINARY_LIMITS_IS_64_FLAG)
 
@@ -89,12 +94,8 @@ enum class NameSectionSubsection {
   Global = 7,
   ElemSegment = 8,
   DataSegment = 9,
-  // tag names are yet part of the extended-name-section proposal (because it
+  Field = 10,
-  // only deals with naming things that are in the spec already).  However, we
+  Tag = 11,
-  // include names for Tags in wabt using this enum value on the basis that tags
-  // can only exist when exceptions are enabled and that engines should ignore
-  // unknown name types.
-  Tag = 10,
 
   First = Module,
   Last = Tag,

third_party/wasm2c/include/wabt/c-writer.h

@@ -29,6 +29,7 @@ class Stream;
 
 struct WriteCOptions {
   std::string_view module_name;
+  Features features;
   /*
    * name_to_output_file_index takes const iterators to begin and end of a list
    * of all functions in the module, number of imported functions, and number of

third_party/wasm2c/include/wabt/circular-array.h

@@ -1,123 +0,0 @@
-/*
- * Copyright 2017 WebAssembly Community Group participants
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef WABT_CIRCULAR_ARRAY_H_
-#define WABT_CIRCULAR_ARRAY_H_
-
-#include <array>
-#include <cassert>
-#include <cstddef>
-#include <type_traits>
-
-namespace wabt {
-
-// TODO(karlschimpf) Complete the API
-// Note: Capacity must be a power of 2.
-template <class T, size_t kCapacity>
-class CircularArray {
- public:
-  using value_type = T;
-  using reference = value_type&;
-  using const_reference = const value_type&;
-  using size_type = size_t;
-  using difference_type = ptrdiff_t;
-
-  CircularArray() {
-    static_assert(kCapacity && ((kCapacity & (kCapacity - 1)) == 0),
-                  "Capacity must be a power of 2.");
-  }
-
-  CircularArray(const CircularArray&) = default;
-  CircularArray& operator=(const CircularArray&) = default;
-
-  CircularArray(CircularArray&&) = default;
-  CircularArray& operator=(CircularArray&&) = default;
-
-  ~CircularArray() { clear(); }
-
-  reference at(size_type index) {
-    assert(index < size_);
-    return (*this)[index];
-  }
-
-  const_reference at(size_type index) const {
-    assert(index < size_);
-    return (*this)[index];
-  }
-
-  reference operator[](size_type index) { return contents_[position(index)]; }
-
-  const_reference operator[](size_type index) const {
-    return contents_[position(index)];
-  }
-
-  reference back() { return at(size_ - 1); }
-
-  const_reference back() const { return at(size_ - 1); }
-
-  bool empty() const { return size_ == 0; }
-
-  reference front() { return at(0); }
-
-  const_reference front() const { return at(0); }
-
-  size_type max_size() const { return kCapacity; }
-
-  void pop_back() {
-    assert(size_ > 0);
-    SetElement(back());
-    --size_;
-  }
-
-  void pop_front() {
-    assert(size_ > 0);
-    SetElement(front());
-    front_ = (front_ + 1) & kMask;
-    --size_;
-  }
-
-  void push_back(const value_type& value) {
-    assert(size_ < kCapacity);
-    SetElement(at(size_++), value);
-  }
-
-  size_type size() const { return size_; }
-
-  void clear() {
-    while (!empty()) {
-      pop_back();
-    }
-  }
-
- private:
-  static constexpr size_type kMask = kCapacity - 1;
-
-  size_t position(size_t index) const { return (front_ + index) & kMask; }
-
-  template <typename... Args>
-  void SetElement(reference element, Args&&... args) {
-    element.~T();
-    new (&element) T(std::forward<Args>(args)...);
-  }
-
-  std::array<T, kCapacity> contents_;
-  size_type size_ = 0;
-  size_type front_ = 0;
-};
-
-}  // namespace wabt
-
-#endif  // WABT_CIRCULAR_ARRAY_H_

third_party/wasm2c/include/wabt/common.h

@@ -44,14 +44,18 @@
 #define WABT_USE(x) static_cast<void>(x)
 
 // 64k
-#define WABT_PAGE_SIZE 0x10000
+#define WABT_DEFAULT_PAGE_SIZE 0x10000
-// # of pages that fit in 32-bit address space
+
-#define WABT_MAX_PAGES32 0x10000
+inline uint64_t WABT_BYTES_TO_MIN_PAGES(uint64_t num_bytes,
-// # of pages that fit in 64-bit address space
+                                        uint32_t page_size) {
-#define WABT_MAX_PAGES64 0x1000000000000
+  if ((page_size == 0) ||
-#define WABT_BYTES_TO_PAGES(x) ((x) >> 16)
+      (page_size & (page_size - 1))) {  // malformed page sizes
-#define WABT_ALIGN_UP_TO_PAGE(x) \
+    WABT_UNREACHABLE;
-  (((x) + WABT_PAGE_SIZE - 1) & ~(WABT_PAGE_SIZE - 1))
+    return 0;
+  }
+  uint64_t num_pages = num_bytes / page_size;
+  return (page_size * num_pages == num_bytes) ? num_pages : num_pages + 1;
+}
 
 #define WABT_ENUM_COUNT(name) \
   (static_cast<int>(name::Last) - static_cast<int>(name::First) + 1)
@@ -166,7 +170,7 @@ Dst WABT_VECTORCALL Bitcast(Src&& value) {
 
 template <typename T>
 void ZeroMemory(T& v) {
-  WABT_STATIC_ASSERT(std::is_pod<T>::value);
+  WABT_STATIC_ASSERT(std::is_trivial<T>::value);
   memset(&v, 0, sizeof(v));
 }
 
@@ -277,10 +281,14 @@ enum class RelocType {
   TableIndexI64 = 19,           // Memory64: Like TableIndexI32
   TableNumberLEB = 20,          // e.g. Immediate of table.get
   MemoryAddressTLSSLEB = 21,    // Address relative to __tls_base
-  MemoryAddressTLSI32 = 22,     // Address relative to __tls_base
+  FunctionOffsetI64 = 22,       // Memory64: Like FunctionOffsetI32
+  MemoryAddressLocRelI32 = 23,  // Address relative to the relocation's location
+  TableIndexRelSLEB64 = 24,     // Memory64: TableIndexRelSLEB
+  MemoryAddressTLSSLEB64 = 25,  // Memory64: MemoryAddressTLSSLEB
+  FuncIndexI32 = 26,            // Function index as an I32
 
   First = FuncIndexLEB,
-  Last = MemoryAddressTLSI32,
+  Last = FuncIndexI32,
 };
 constexpr int kRelocTypeCount = WABT_ENUM_COUNT(RelocType);
 
@@ -328,10 +336,12 @@ enum class ComdatType {
 #define WABT_SYMBOL_FLAG_EXPLICIT_NAME 0x40
 #define WABT_SYMBOL_FLAG_NO_STRIP 0x80
 #define WABT_SYMBOL_FLAG_TLS 0x100
-#define WABT_SYMBOL_FLAG_MAX 0x1ff
+#define WABT_SYMBOL_FLAG_ABS 0x200
+#define WABT_SYMBOL_FLAG_MAX 0x3ff
 
 #define WABT_SEGMENT_FLAG_STRINGS 0x1
 #define WABT_SEGMENT_FLAG_TLS 0x2
+#define WASM_SEGMENT_FLAG_RETAIN 0x4
 #define WABT_SEGMENT_FLAG_MAX 0xff
 
 enum class SymbolVisibility {

third_party/wasm2c/include/wabt/feature.def

@@ -40,3 +40,4 @@ WABT_FEATURE(memory64,            "memory64",                false,   "64-bit me
 WABT_FEATURE(multi_memory,        "multi-memory",            false,   "Multi-memory")
 WABT_FEATURE(extended_const,      "extended-const",          false,   "Extended constant expressions")
 WABT_FEATURE(relaxed_simd,        "relaxed-simd",            false,   "Relaxed SIMD")
+WABT_FEATURE(custom_page_sizes,   "custom-page-sizes",       false,   "Custom page sizes")

third_party/wasm2c/include/wabt/ir.h

@@ -25,6 +25,7 @@
 #include <string_view>
 #include <type_traits>
 #include <vector>
+#include <set>
 
 #include "wabt/binding-hash.h"
 #include "wabt/common.h"
@@ -310,6 +311,13 @@ class FuncType : public TypeEntry {
   Type GetResultType(Index index) const { return sig.GetResultType(index); }
 
   FuncSignature sig;
+
+  // The BinaryReaderIR tracks whether a FuncType is the target of a tailcall
+  // (via a return_call_indirect). wasm2c (CWriter) uses this information to
+  // limit its output in some cases.
+  struct {
+    bool tailcall = false;
+  } features_used;
 };
 
 struct Field {
@@ -493,15 +501,15 @@ class MemoryExpr : public ExprMixin<TypeEnum> {
 template <ExprType TypeEnum>
 class MemoryBinaryExpr : public ExprMixin<TypeEnum> {
  public:
-  MemoryBinaryExpr(Var srcmemidx,
+  MemoryBinaryExpr(Var destmemidx,
-                   Var destmemidx,
+                   Var srcmemidx,
                    const Location& loc = Location())
       : ExprMixin<TypeEnum>(loc),
-        srcmemidx(srcmemidx),
+        destmemidx(destmemidx),
-        destmemidx(destmemidx) {}
+        srcmemidx(srcmemidx) {}
 
-  Var srcmemidx;
   Var destmemidx;
+  Var srcmemidx;
 };
 
 using DropExpr = ExprMixin<ExprType::Drop>;
@@ -893,6 +901,13 @@ struct Func {
   BindingHash bindings;
   ExprList exprs;
   Location loc;
+
+  // For a subset of features, the BinaryReaderIR tracks whether they are
+  // actually used by the function. wasm2c (CWriter) uses this information to
+  // limit its output in some cases.
+  struct {
+    bool tailcall = false;
+  } features_used;
 };
 
 struct Global {
@@ -932,6 +947,7 @@ struct Memory {
 
   std::string name;
   Limits page_limits;
+  uint32_t page_size;
 };
 
 struct DataSegment {
@@ -1164,6 +1180,17 @@ class StartModuleField : public ModuleFieldMixin<ModuleFieldType::Start> {
   Var start;
 };
 
+struct Custom {
+  explicit Custom(const Location& loc = Location(),
+                  std::string_view name = std::string_view(),
+                  std::vector<uint8_t> data = std::vector<uint8_t>())
+      : name(name), data(data), loc(loc) {}
+
+  std::string name;
+  std::vector<uint8_t> data;
+  Location loc;
+};
+
 struct Module {
   Index GetFuncTypeIndex(const Var&) const;
   Index GetFuncTypeIndex(const FuncDeclaration&) const;
@@ -1235,6 +1262,7 @@ struct Module {
   std::vector<Memory*> memories;
   std::vector<DataSegment*> data_segments;
   std::vector<Var*> starts;
+  std::vector<Custom> customs;
 
   BindingHash tag_bindings;
   BindingHash func_bindings;
@@ -1252,7 +1280,14 @@ struct Module {
   struct {
     bool simd = false;
     bool exceptions = false;
+    bool threads = false;
   } features_used;
+
+  // The BinaryReaderIR tracks function references used by the module, whether
+  // in element segment initializers, global initializers, or functions. wasm2c
+  // needs to emit wrappers for any functions that might get used as function
+  // references, and uses this information to limit its output.
+  std::set<Index> used_func_refs;
 };
 
 enum class ScriptModuleType {

third_party/wasm2c/include/wabt/opcode.def

@@ -164,13 +164,13 @@ WABT_OPCODE(I64,  I64,  I64,  ___,  0,  0,    0x87, I64ShrS, "i64.shr_s", ">>")
 WABT_OPCODE(I64,  I64,  I64,  ___,  0,  0,    0x88, I64ShrU, "i64.shr_u", ">>")
 WABT_OPCODE(I64,  I64,  I64,  ___,  0,  0,    0x89, I64Rotl, "i64.rotl", "<<")
 WABT_OPCODE(I64,  I64,  I64,  ___,  0,  0,    0x8a, I64Rotr, "i64.rotr", ">>")
-WABT_OPCODE(F32,  F32,  F32,  ___,  0,  0,    0x8b, F32Abs, "f32.abs", "abs")
+WABT_OPCODE(F32,  F32,  ___,  ___,  0,  0,    0x8b, F32Abs, "f32.abs", "abs")
-WABT_OPCODE(F32,  F32,  F32,  ___,  0,  0,    0x8c, F32Neg, "f32.neg", "-")
+WABT_OPCODE(F32,  F32,  ___,  ___,  0,  0,    0x8c, F32Neg, "f32.neg", "-")
-WABT_OPCODE(F32,  F32,  F32,  ___,  0,  0,    0x8d, F32Ceil, "f32.ceil", "ceil")
+WABT_OPCODE(F32,  F32,  ___,  ___,  0,  0,    0x8d, F32Ceil, "f32.ceil", "ceil")
-WABT_OPCODE(F32,  F32,  F32,  ___,  0,  0,    0x8e, F32Floor, "f32.floor", "floor")
+WABT_OPCODE(F32,  F32,  ___,  ___,  0,  0,    0x8e, F32Floor, "f32.floor", "floor")
-WABT_OPCODE(F32,  F32,  F32,  ___,  0,  0,    0x8f, F32Trunc, "f32.trunc", "trunc")
+WABT_OPCODE(F32,  F32,  ___,  ___,  0,  0,    0x8f, F32Trunc, "f32.trunc", "trunc")
-WABT_OPCODE(F32,  F32,  F32,  ___,  0,  0,    0x90, F32Nearest, "f32.nearest", "nearest")
+WABT_OPCODE(F32,  F32,  ___,  ___,  0,  0,    0x90, F32Nearest, "f32.nearest", "nearest")
-WABT_OPCODE(F32,  F32,  F32,  ___,  0,  0,    0x91, F32Sqrt, "f32.sqrt", "sqrt")
+WABT_OPCODE(F32,  F32,  ___,  ___,  0,  0,    0x91, F32Sqrt, "f32.sqrt", "sqrt")
 WABT_OPCODE(F32,  F32,  F32,  ___,  0,  0,    0x92, F32Add, "f32.add", "+")
 WABT_OPCODE(F32,  F32,  F32,  ___,  0,  0,    0x93, F32Sub, "f32.sub", "-")
 WABT_OPCODE(F32,  F32,  F32,  ___,  0,  0,    0x94, F32Mul, "f32.mul", "*")
@@ -178,13 +178,13 @@ WABT_OPCODE(F32,  F32,  F32,  ___,  0,  0,    0x95, F32Div, "f32.div", "/")
 WABT_OPCODE(F32,  F32,  F32,  ___,  0,  0,    0x96, F32Min, "f32.min", "min")
 WABT_OPCODE(F32,  F32,  F32,  ___,  0,  0,    0x97, F32Max, "f32.max", "max")
 WABT_OPCODE(F32,  F32,  F32,  ___,  0,  0,    0x98, F32Copysign, "f32.copysign", "copysign")
-WABT_OPCODE(F64,  F64,  F64,  ___,  0,  0,    0x99, F64Abs, "f64.abs", "abs")
+WABT_OPCODE(F64,  F64,  ___,  ___,  0,  0,    0x99, F64Abs, "f64.abs", "abs")
-WABT_OPCODE(F64,  F64,  F64,  ___,  0,  0,    0x9a, F64Neg, "f64.neg", "-")
+WABT_OPCODE(F64,  F64,  ___,  ___,  0,  0,    0x9a, F64Neg, "f64.neg", "-")
-WABT_OPCODE(F64,  F64,  F64,  ___,  0,  0,    0x9b, F64Ceil, "f64.ceil", "ceil")
+WABT_OPCODE(F64,  F64,  ___,  ___,  0,  0,    0x9b, F64Ceil, "f64.ceil", "ceil")
-WABT_OPCODE(F64,  F64,  F64,  ___,  0,  0,    0x9c, F64Floor, "f64.floor", "floor")
+WABT_OPCODE(F64,  F64,  ___,  ___,  0,  0,    0x9c, F64Floor, "f64.floor", "floor")
-WABT_OPCODE(F64,  F64,  F64,  ___,  0,  0,    0x9d, F64Trunc, "f64.trunc", "trunc")
+WABT_OPCODE(F64,  F64,  ___,  ___,  0,  0,    0x9d, F64Trunc, "f64.trunc", "trunc")
-WABT_OPCODE(F64,  F64,  F64,  ___,  0,  0,    0x9e, F64Nearest, "f64.nearest", "nearest")
+WABT_OPCODE(F64,  F64,  ___,  ___,  0,  0,    0x9e, F64Nearest, "f64.nearest", "nearest")
-WABT_OPCODE(F64,  F64,  F64,  ___,  0,  0,    0x9f, F64Sqrt, "f64.sqrt", "sqrt")
+WABT_OPCODE(F64,  F64,  ___,  ___,  0,  0,    0x9f, F64Sqrt, "f64.sqrt", "sqrt")
 WABT_OPCODE(F64,  F64,  F64,  ___,  0,  0,    0xa0, F64Add, "f64.add", "+")
 WABT_OPCODE(F64,  F64,  F64,  ___,  0,  0,    0xa1, F64Sub, "f64.sub", "-")
 WABT_OPCODE(F64,  F64,  F64,  ___,  0,  0,    0xa2, F64Mul, "f64.mul", "*")
@@ -247,7 +247,7 @@ WABT_OPCODE(I64,  F64,  ___,  ___,  0,  0xfc, 0x07, I64TruncSatF64U, "i64.trunc_
 /* Bulk-memory */
 WABT_OPCODE(___, I32,  I32,  I32,  0,  0xfc, 0x08, MemoryInit, "memory.init", "")
 WABT_OPCODE(___, ___,  ___,  ___,  0,  0xfc, 0x09, DataDrop, "data.drop", "")
-WABT_OPCODE(___, I32,  I32,  I32,  0,  0xfc, 0x0a, MemoryCopy,"memory.copy", "")
+WABT_OPCODE(___, I32,  I32,  I32,  0,  0xfc, 0x0a, MemoryCopy, "memory.copy", "")
 WABT_OPCODE(___, I32,  I32,  I32,  0,  0xfc, 0x0b, MemoryFill, "memory.fill", "")
 WABT_OPCODE(___, I32,  I32,  I32,  0,  0xfc, 0x0c, TableInit, "table.init", "")
 WABT_OPCODE(___, ___,  ___,  ___,  0,  0xfc, 0x0d, ElemDrop, "elem.drop", "")
@@ -264,7 +264,7 @@ WABT_OPCODE(___,  ___,  ___,  ___,  0,  0,    0xd1, RefIsNull, "ref.is_null", ""
 WABT_OPCODE(___,  ___,  ___,  ___,  0,  0,    0xd2, RefFunc, "ref.func", "")
 
 /* Simd opcodes */
-WABT_OPCODE(V128, I32,  ___,  ___,  16, 0xfd, 0x00, V128Load,  "v128.load", "")
+WABT_OPCODE(V128, I32,  ___,  ___,  16, 0xfd, 0x00, V128Load, "v128.load", "")
 WABT_OPCODE(V128, I32,   ___, ___,  8,  0xfd, 0x01, V128Load8X8S, "v128.load8x8_s", "")
 WABT_OPCODE(V128, I32,   ___, ___,  8,  0xfd, 0x02, V128Load8X8U, "v128.load8x8_u", "")
 WABT_OPCODE(V128, I32,   ___, ___,  8,  0xfd, 0x03, V128Load16X4S, "v128.load16x4_s", "")
@@ -492,8 +492,8 @@ WABT_OPCODE(V128, V128, V128, ___,  0,  0xfd, 0xf4, F64X2Min, "f64x2.min", "")
 WABT_OPCODE(V128, V128, V128, ___,  0,  0xfd, 0xf5, F64X2Max, "f64x2.max", "")
 WABT_OPCODE(V128, V128, V128, ___,  0,  0xfd, 0xf6, F64X2PMin, "f64x2.pmin", "")
 WABT_OPCODE(V128, V128, V128, ___,  0,  0xfd, 0xf7, F64X2PMax, "f64x2.pmax", "")
-WABT_OPCODE(V128, V128, ___,  ___,  0,  0xfd, 0xf8, I32X4TruncSatF32X4S,"i32x4.trunc_sat_f32x4_s", "")
+WABT_OPCODE(V128, V128, ___,  ___,  0,  0xfd, 0xf8, I32X4TruncSatF32X4S, "i32x4.trunc_sat_f32x4_s", "")
-WABT_OPCODE(V128, V128, ___,  ___,  0,  0xfd, 0xf9, I32X4TruncSatF32X4U,"i32x4.trunc_sat_f32x4_u", "")
+WABT_OPCODE(V128, V128, ___,  ___,  0,  0xfd, 0xf9, I32X4TruncSatF32X4U, "i32x4.trunc_sat_f32x4_u", "")
 WABT_OPCODE(V128, V128, ___,  ___,  0,  0xfd, 0xfa, F32X4ConvertI32X4S, "f32x4.convert_i32x4_s", "")
 WABT_OPCODE(V128, V128, ___,  ___,  0,  0xfd, 0xfb, F32X4ConvertI32X4U, "f32x4.convert_i32x4_u", "")
 WABT_OPCODE(V128, V128,  ___, ___,  0,  0xfd, 0xfc, I32X4TruncSatF64X2SZero, "i32x4.trunc_sat_f64x2_s_zero", "")
@@ -520,8 +520,8 @@ WABT_OPCODE(V128, V128, V128, ___,  0,  0xfd, 0x10e, F32X4RelaxedMax, "f32x4.rel
 WABT_OPCODE(V128, V128, V128, ___,  0,  0xfd, 0x10f, F64X2RelaxedMin, "f64x2.relaxed_min", "")
 WABT_OPCODE(V128, V128, V128, ___,  0,  0xfd, 0x110, F64X2RelaxedMax, "f64x2.relaxed_max", "")
 WABT_OPCODE(V128, V128, V128, ___,  0,  0xfd, 0x111, I16X8RelaxedQ15mulrS, "i16x8.relaxed_q15mulr_s", "")
-WABT_OPCODE(V128, V128, V128, ___,  0,  0xfd, 0x112, I16X8DotI8X16I7X16S, "i16x8.dot_i8x16_i7x16_s", "")
+WABT_OPCODE(V128, V128, V128, ___,  0,  0xfd, 0x112, I16X8DotI8X16I7X16S, "i16x8.relaxed_dot_i8x16_i7x16_s", "")
-WABT_OPCODE(V128, V128, V128, V128, 0,  0xfd, 0x113, I32X4DotI8X16I7X16AddS, "i32x4.dot_i8x16_i7x16_add_s", "")
+WABT_OPCODE(V128, V128, V128, V128, 0,  0xfd, 0x113, I32X4DotI8X16I7X16AddS, "i32x4.relaxed_dot_i8x16_i7x16_add_s", "")
 
 /* Thread opcodes (--enable-threads) */
 WABT_OPCODE(I32,  I32,  I32,  ___,  4,  0xfe, 0x00, MemoryAtomicNotify, "memory.atomic.notify", "")

third_party/wasm2c/include/wabt/shared-validator.h

@@ -75,7 +75,7 @@ class SharedValidator {
 
   Result OnFunction(const Location&, Var sig_var);
   Result OnTable(const Location&, Type elem_type, const Limits&);
-  Result OnMemory(const Location&, const Limits&);
+  Result OnMemory(const Location&, const Limits&, uint32_t page_size);
   Result OnGlobalImport(const Location&, Type type, bool mutable_);
   Result OnGlobal(const Location&, Type type, bool mutable_);
   Result OnTag(const Location&, Var sig_var);
@@ -89,9 +89,6 @@ class SharedValidator {
 
   Result OnElemSegment(const Location&, Var table_var, SegmentKind);
   Result OnElemSegmentElemType(const Location&, Type elem_type);
-  Result OnElemSegmentElemExpr_RefNull(const Location&, Type type);
-  Result OnElemSegmentElemExpr_RefFunc(const Location&, Var func_var);
-  Result OnElemSegmentElemExpr_Other(const Location&);
 
   void OnDataCount(Index count);
   Result OnDataSegment(const Location&, Var memory_var, SegmentKind);
@@ -104,12 +101,36 @@ class SharedValidator {
   Result OnLocalDecl(const Location&, Index count, Type type);
 
   Result OnAtomicFence(const Location&, uint32_t consistency_model);
-  Result OnAtomicLoad(const Location&, Opcode, Var memidx, Address align);
+  Result OnAtomicLoad(const Location&,
-  Result OnAtomicNotify(const Location&, Opcode, Var memidx, Address align);
+                      Opcode,
-  Result OnAtomicRmwCmpxchg(const Location&, Opcode, Var memidx, Address align);
+                      Var memidx,
-  Result OnAtomicRmw(const Location&, Opcode, Var memidx, Address align);
+                      Address align,
-  Result OnAtomicStore(const Location&, Opcode, Var memidx, Address align);
+                      Address offset);
-  Result OnAtomicWait(const Location&, Opcode, Var memidx, Address align);
+  Result OnAtomicNotify(const Location&,
+                        Opcode,
+                        Var memidx,
+                        Address align,
+                        Address offset);
+  Result OnAtomicRmwCmpxchg(const Location&,
+                            Opcode,
+                            Var memidx,
+                            Address align,
+                            Address offset);
+  Result OnAtomicRmw(const Location&,
+                     Opcode,
+                     Var memidx,
+                     Address align,
+                     Address offset);
+  Result OnAtomicStore(const Location&,
+                       Opcode,
+                       Var memidx,
+                       Address align,
+                       Address offset);
+  Result OnAtomicWait(const Location&,
+                      Opcode,
+                      Var memidx,
+                      Address align,
+                      Address offset);
   Result OnBinary(const Location&, Opcode);
   Result OnBlock(const Location&, Type sig_type);
   Result OnBr(const Location&, Var depth);
@@ -133,14 +154,22 @@ class SharedValidator {
   Result OnGlobalGet(const Location&, Var);
   Result OnGlobalSet(const Location&, Var);
   Result OnIf(const Location&, Type sig_type);
-  Result OnLoad(const Location&, Opcode, Var memidx, Address align);
+  Result OnLoad(const Location&, Opcode, Var memidx, Address align, Address offset);
-  Result OnLoadSplat(const Location&, Opcode, Var memidx, Address align);
+  Result OnLoadSplat(const Location&,
-  Result OnLoadZero(const Location&, Opcode, Var memidx, Address align);
+                     Opcode,
+                     Var memidx,
+                     Address align,
+                     Address offset);
+  Result OnLoadZero(const Location&,
+                    Opcode,
+                    Var memidx,
+                    Address align,
+                    Address offset);
   Result OnLocalGet(const Location&, Var);
   Result OnLocalSet(const Location&, Var);
   Result OnLocalTee(const Location&, Var);
   Result OnLoop(const Location&, Type sig_type);
-  Result OnMemoryCopy(const Location&, Var srcmemidx, Var destmemidx);
+  Result OnMemoryCopy(const Location&, Var destmemidx, Var srcmemidx);
   Result OnMemoryFill(const Location&, Var memidx);
   Result OnMemoryGrow(const Location&, Var memidx);
   Result OnMemoryInit(const Location&, Var segment_var, Var memidx);
@@ -159,14 +188,20 @@ class SharedValidator {
                         Opcode,
                         Var memidx,
                         Address align,
+                        Address offset,
                         uint64_t lane_idx);
   Result OnSimdStoreLane(const Location&,
                          Opcode,
                          Var memidx,
                          Address align,
+                         Address offset,
                          uint64_t lane_idx);
   Result OnSimdShuffleOp(const Location&, Opcode, v128 lane_idx);
-  Result OnStore(const Location&, Opcode, Var memidx, Address align);
+  Result OnStore(const Location&,
+                 Opcode,
+                 Var memidx,
+                 Address align,
+                 Address offset);
   Result OnTableCopy(const Location&, Var dst_var, Var src_var);
   Result OnTableFill(const Location&, Var table_var);
   Result OnTableGet(const Location&, Var table_var);
@@ -280,6 +315,7 @@ class SharedValidator {
   Result CheckDataSegmentIndex(Var data_segment_var);
 
   Result CheckAlign(const Location&, Address align, Address natural_align);
+  Result CheckOffset(const Location&, Address offset, const Limits& limits);
   Result CheckAtomicAlign(const Location&,
                           Address align,
                           Address natural_align);

third_party/wasm2c/include/wabt/token.def

@@ -20,6 +20,7 @@
 
 /* Tokens with no additional data (i.e. bare). */
 WABT_TOKEN(Invalid, "Invalid")
+WABT_TOKEN(After, "after")
 WABT_TOKEN(Array, "array")
 WABT_TOKEN(AssertException, "assert_exception")
 WABT_TOKEN(AssertExhaustion, "assert_exhaustion")
@@ -28,6 +29,7 @@ WABT_TOKEN(AssertMalformed, "assert_malformed")
 WABT_TOKEN(AssertReturn, "assert_return")
 WABT_TOKEN(AssertTrap, "assert_trap")
 WABT_TOKEN(AssertUnlinkable, "assert_unlinkable")
+WABT_TOKEN(Before, "before")
 WABT_TOKEN(Bin, "bin")
 WABT_TOKEN(Item, "item")
 WABT_TOKEN(Data, "data")
@@ -40,6 +42,7 @@ WABT_TOKEN(Eof, "EOF")
 WABT_TOKEN(Tag, "tag")
 WABT_TOKEN(Export, "export")
 WABT_TOKEN(Field, "field")
+WABT_TOKEN(Function, "function")
 WABT_TOKEN(Get, "get")
 WABT_TOKEN(Global, "global")
 WABT_TOKEN(Import, "import")
@@ -54,6 +57,7 @@ WABT_TOKEN(NanArithmetic, "nan:arithmetic")
 WABT_TOKEN(NanCanonical, "nan:canonical")
 WABT_TOKEN(Offset, "offset")
 WABT_TOKEN(Output, "output")
+WABT_TOKEN(PageSize, "pagesize")
 WABT_TOKEN(Param, "param")
 WABT_TOKEN(Ref, "ref")
 WABT_TOKEN(Quote, "quote")
@@ -95,6 +99,7 @@ WABT_TOKEN(Block, "block")
 WABT_TOKEN(Br, "br")
 WABT_TOKEN(BrIf, "br_if")
 WABT_TOKEN(BrTable, "br_table")
+WABT_TOKEN(Code, "code")
 WABT_TOKEN(Call, "call")
 WABT_TOKEN(CallIndirect, "call_indirect")
 WABT_TOKEN(CallRef, "call_ref")
@@ -138,7 +143,7 @@ WABT_TOKEN(SimdStoreLane, "SIMDSTORELANE")
 WABT_TOKEN(SimdShuffleOp, "i8x16.shuffle")
 WABT_TOKEN(Store, "STORE")
 WABT_TOKEN(TableCopy, "table.copy")
-WABT_TOKEN(TableFill, "table.full")
+WABT_TOKEN(TableFill, "table.fill")
 WABT_TOKEN(TableGet, "table.get")
 WABT_TOKEN(TableGrow, "table.grow")
 WABT_TOKEN(TableInit, "table.init")

third_party/wasm2c/include/wabt/type-checker.h

@@ -18,6 +18,7 @@
 #define WABT_TYPE_CHECKER_H_
 
 #include <functional>
+#include <type_traits>
 #include <vector>
 
 #include "wabt/common.h"
@@ -77,7 +78,8 @@ class TypeChecker {
   Result EndBrTable();
   Result OnCall(const TypeVector& param_types, const TypeVector& result_types);
   Result OnCallIndirect(const TypeVector& param_types,
-                        const TypeVector& result_types);
+                        const TypeVector& result_types,
+                        const Limits& table_limits);
   Result OnIndexedFuncRef(Index* out_index);
   Result OnReturnCall(const TypeVector& param_types,
                       const TypeVector& result_types);
@@ -99,21 +101,21 @@ class TypeChecker {
   Result OnLocalSet(Type);
   Result OnLocalTee(Type);
   Result OnLoop(const TypeVector& param_types, const TypeVector& result_types);
-  Result OnMemoryCopy(const Limits& srclimits, const Limits& dstlimits);
+  Result OnMemoryCopy(const Limits& dst_limits, const Limits& src_limits);
   Result OnDataDrop(Index);
   Result OnMemoryFill(const Limits& limits);
   Result OnMemoryGrow(const Limits& limits);
   Result OnMemoryInit(Index, const Limits& limits);
   Result OnMemorySize(const Limits& limits);
-  Result OnTableCopy();
+  Result OnTableCopy(const Limits& dst_limits, const Limits& src_limits);
   Result OnElemDrop(Index);
-  Result OnTableInit(Index, Index);
+  Result OnTableInit(Index, const Limits& limits);
-  Result OnTableGet(Type elem_type);
+  Result OnTableGet(Type elem_type, const Limits& limits);
-  Result OnTableSet(Type elem_type);
+  Result OnTableSet(Type elem_type, const Limits& limits);
-  Result OnTableGrow(Type elem_type);
+  Result OnTableGrow(Type elem_type, const Limits& limits);
-  Result OnTableSize();
+  Result OnTableSize(const Limits& limits);
-  Result OnTableFill(Type elem_type);
+  Result OnTableFill(Type elem_type, const Limits& limits);
-  Result OnRefFuncExpr(Index func_type);
+  Result OnRefFuncExpr(Index func_type, bool force_generic_funcref);
   Result OnRefNullExpr(Type type);
   Result OnRefIsNullExpr();
   Result OnRethrow(Index depth);
@@ -181,6 +183,9 @@ class TypeChecker {
 
   template <typename... Args>
   void PrintStackIfFailed(Result result, const char* desc, Args... args) {
+    // Assert all args are Type or Type::Enum. If it's a TypeVector then
+    // PrintStackIfFailedV() should be used instead.
+    static_assert((std::is_constructible_v<Type, Args> && ...));
     // Minor optimization, check result before constructing the vector to pass
     // to the other overload of PrintStackIfFailed.
     if (Failed(result)) {

third_party/wasm2c/include/wabt/type.h

@@ -42,6 +42,7 @@ class Type {
     V128 = -0x05,       // 0x7b
     I8 = -0x06,         // 0x7a  : packed-type only, used in gc and as v128 lane
     I16 = -0x07,        // 0x79  : packed-type only, used in gc and as v128 lane
+    ExnRef = -0x17,     // 0x69
     FuncRef = -0x10,    // 0x70
     ExternRef = -0x11,  // 0x6f
     Reference = -0x15,  // 0x6b
@@ -68,7 +69,7 @@ class Type {
 
   bool IsRef() const {
     return enum_ == Type::ExternRef || enum_ == Type::FuncRef ||
-           enum_ == Type::Reference;
+           enum_ == Type::Reference || enum_ == Type::ExnRef;
   }
 
   bool IsReferenceWithIndex() const { return enum_ == Type::Reference; }
@@ -87,6 +88,7 @@ class Type {
       case Type::V128:      return "v128";
       case Type::I8:        return "i8";
       case Type::I16:       return "i16";
+      case Type::ExnRef:    return "exnref";
       case Type::FuncRef:   return "funcref";
       case Type::Func:      return "func";
       case Type::Void:      return "void";
@@ -103,6 +105,7 @@ class Type {
     switch (enum_) {
       case Type::FuncRef:   return "func";
       case Type::ExternRef: return "extern";
+      case Type::ExnRef:    return "exn";
       case Type::Struct:    return "struct";
       case Type::Array:     return "array";
       default:              return "<invalid>";
@@ -145,6 +148,7 @@ class Type {
       case Type::F64:
       case Type::V128:
       case Type::FuncRef:
+      case Type::ExnRef:
       case Type::ExternRef:
       case Type::Reference:
         return TypeVector(this, this + 1);

third_party/wasm2c/include/wabt/wast-parser.h

@@ -19,9 +19,9 @@
 
 #include <array>
 #include <memory>
+#include <optional>
 #include <unordered_map>
 
-#include "wabt/circular-array.h"
 #include "wabt/error.h"
 #include "wabt/feature.h"
 #include "wabt/intrusive-list.h"
@@ -147,6 +147,7 @@ class WastParser {
   Result ParseMemidx(Location loc, Var* memidx);
   Result ParseLimitsIndex(Limits*);
   Result ParseLimits(Limits*);
+  Result ParsePageSize(uint32_t*);
   Result ParseNat(uint64_t*, bool is_64);
 
   Result ParseModuleFieldList(Module*);
@@ -163,6 +164,9 @@ class WastParser {
   Result ParseStartModuleField(Module*);
   Result ParseTableModuleField(Module*);
 
+  Result ParseCustomSectionAnnotation(Module*);
+  bool PeekIsCustom();
+
   Result ParseExportDesc(Export*);
   Result ParseInlineExports(ModuleFieldList*, ExternalKind);
   Result ParseInlineImport(Import*);
@@ -252,13 +256,28 @@ class WastParser {
   Result ParseSimdV128Const(Const*, TokenType, ConstType);
 
   void CheckImportOrdering(Module*);
+  bool HasError() const;
 
   WastLexer* lexer_;
   Index last_module_index_ = kInvalidIndex;
   Errors* errors_;
   WastParseOptions* options_;
 
-  CircularArray<Token, 2> tokens_;
+  // two-element queue of upcoming tokens
+  class TokenQueue {
+    std::array<std::optional<Token>, 2> tokens{};
+    bool i{};
+
+   public:
+    void push_back(Token t);
+    void pop_front();
+    const Token& at(size_t n) const;
+    const Token& front() const;
+    bool empty() const;
+    size_t size() const;
+  };
+
+  TokenQueue tokens_{};
 };
 
 Result ParseWatModule(WastLexer* lexer,

third_party/wasm2c/src/apply-names.cc

@@ -263,8 +263,8 @@ Result NameApplier::OnDataDropExpr(DataDropExpr* expr) {
 }
 
 Result NameApplier::OnMemoryCopyExpr(MemoryCopyExpr* expr) {
-  CHECK_RESULT(UseNameForMemoryVar(&expr->srcmemidx));
   CHECK_RESULT(UseNameForMemoryVar(&expr->destmemidx));
+  CHECK_RESULT(UseNameForMemoryVar(&expr->srcmemidx));
   return Result::Ok;
 }
 

third_party/wasm2c/src/binary-reader-ir.cc

@@ -126,7 +126,8 @@ class BinaryReaderIR : public BinaryReaderNop {
                         std::string_view module_name,
                         std::string_view field_name,
                         Index memory_index,
-                        const Limits* page_limits) override;
+                        const Limits* page_limits,
+                        uint32_t page_size) override;
   Result OnImportGlobal(Index import_index,
                         std::string_view module_name,
                         std::string_view field_name,
@@ -148,7 +149,9 @@ class BinaryReaderIR : public BinaryReaderNop {
                  const Limits* elem_limits) override;
 
   Result OnMemoryCount(Index count) override;
-  Result OnMemory(Index index, const Limits* limits) override;
+  Result OnMemory(Index index,
+                  const Limits* limits,
+                  uint32_t page_size) override;
 
   Result OnGlobalCount(Index count) override;
   Result BeginGlobal(Index index, Type type, bool mutable_) override;
@@ -229,7 +232,7 @@ class BinaryReaderIR : public BinaryReaderNop {
   Result OnLocalSetExpr(Index local_index) override;
   Result OnLocalTeeExpr(Index local_index) override;
   Result OnLoopExpr(Type sig_type) override;
-  Result OnMemoryCopyExpr(Index srcmemidx, Index destmemidx) override;
+  Result OnMemoryCopyExpr(Index destmemidx, Index srcmemidx) override;
   Result OnDataDropExpr(Index segment_index) override;
   Result OnMemoryFillExpr(Index memidx) override;
   Result OnMemoryGrowExpr(Index memidx) override;
@@ -289,9 +292,8 @@ class BinaryReaderIR : public BinaryReaderNop {
   Result EndElemSegmentInitExpr(Index index) override;
   Result OnElemSegmentElemType(Index index, Type elem_type) override;
   Result OnElemSegmentElemExprCount(Index index, Index count) override;
-  Result OnElemSegmentElemExpr_RefNull(Index segment_index, Type type) override;
+  Result BeginElemExpr(Index elem_index, Index expr_index) override;
-  Result OnElemSegmentElemExpr_RefFunc(Index segment_index,
+  Result EndElemExpr(Index elem_index, Index expr_index) override;
-                                       Index func_index) override;
 
   Result OnDataSegmentCount(Index count) override;
   Result BeginDataSegment(Index index,
@@ -315,6 +317,10 @@ class BinaryReaderIR : public BinaryReaderNop {
                      Index index,
                      std::string_view name) override;
 
+  Result OnGenericCustomSection(std::string_view name,
+                                const void* data,
+                                Offset size) override;
+
   Result BeginTagSection(Offset size) override { return Result::Ok; }
   Result OnTagCount(Index count) override { return Result::Ok; }
   Result OnTagType(Index index, Index sig_index) override;
@@ -534,6 +540,13 @@ Result BinaryReaderIR::OnFuncType(Index index,
       std::any_of(func_type->sig.result_types.begin(),
                   func_type->sig.result_types.end(),
                   [](auto x) { return x == Type::V128; });
+  module_->features_used.exceptions |=
+      std::any_of(func_type->sig.param_types.begin(),
+                  func_type->sig.param_types.end(),
+                  [](auto x) { return x == Type::ExnRef; }) ||
+      std::any_of(func_type->sig.result_types.begin(),
+                  func_type->sig.result_types.end(),
+                  [](auto x) { return x == Type::ExnRef; });
 
   field->type = std::move(func_type);
   module_->AppendField(std::move(field));
@@ -550,6 +563,7 @@ Result BinaryReaderIR::OnStructType(Index index,
     struct_type->fields[i].type = fields[i].type;
     struct_type->fields[i].mutable_ = fields[i].mutable_;
     module_->features_used.simd |= (fields[i].type == Type::V128);
+    module_->features_used.exceptions |= (fields[i].type == Type::ExnRef);
   }
   field->type = std::move(struct_type);
   module_->AppendField(std::move(field));
@@ -562,6 +576,7 @@ Result BinaryReaderIR::OnArrayType(Index index, TypeMut type_mut) {
   array_type->field.type = type_mut.type;
   array_type->field.mutable_ = type_mut.mutable_;
   module_->features_used.simd |= (type_mut.type == Type::V128);
+  module_->features_used.exceptions |= (type_mut.type == Type::ExnRef);
   field->type = std::move(array_type);
   module_->AppendField(std::move(field));
   return Result::Ok;
@@ -608,11 +623,16 @@ Result BinaryReaderIR::OnImportMemory(Index import_index,
                                       std::string_view module_name,
                                       std::string_view field_name,
                                       Index memory_index,
-                                      const Limits* page_limits) {
+                                      const Limits* page_limits,
+                                      uint32_t page_size) {
   auto import = std::make_unique<MemoryImport>();
   import->module_name = module_name;
   import->field_name = field_name;
   import->memory.page_limits = *page_limits;
+  import->memory.page_size = page_size;
+  if (import->memory.page_limits.is_shared) {
+    module_->features_used.threads = true;
+  }
   module_->AppendField(
       std::make_unique<ImportModuleField>(std::move(import), GetLocation()));
   return Result::Ok;
@@ -632,6 +652,7 @@ Result BinaryReaderIR::OnImportGlobal(Index import_index,
   module_->AppendField(
       std::make_unique<ImportModuleField>(std::move(import), GetLocation()));
   module_->features_used.simd |= (type == Type::V128);
+  module_->features_used.exceptions |= (type == Type::ExnRef);
   return Result::Ok;
 }
 
@@ -679,6 +700,7 @@ Result BinaryReaderIR::OnTable(Index index,
   Table& table = field->table;
   table.elem_limits = *elem_limits;
   table.elem_type = elem_type;
+  module_->features_used.exceptions |= (elem_type == Type::ExnRef);
   module_->AppendField(std::move(field));
   return Result::Ok;
 }
@@ -690,10 +712,16 @@ Result BinaryReaderIR::OnMemoryCount(Index count) {
   return Result::Ok;
 }
 
-Result BinaryReaderIR::OnMemory(Index index, const Limits* page_limits) {
+Result BinaryReaderIR::OnMemory(Index index,
+                                const Limits* page_limits,
+                                uint32_t page_size) {
   auto field = std::make_unique<MemoryModuleField>(GetLocation());
   Memory& memory = field->memory;
   memory.page_limits = *page_limits;
+  memory.page_size = page_size;
+  if (memory.page_limits.is_shared) {
+    module_->features_used.threads = true;
+  }
   module_->AppendField(std::move(field));
   return Result::Ok;
 }
@@ -712,6 +740,7 @@ Result BinaryReaderIR::BeginGlobal(Index index, Type type, bool mutable_) {
   global.mutable_ = mutable_;
   module_->AppendField(std::move(field));
   module_->features_used.simd |= (type == Type::V128);
+  module_->features_used.exceptions |= (type == Type::ExnRef);
   return Result::Ok;
 }
 
@@ -778,6 +807,7 @@ Result BinaryReaderIR::OnLocalDecl(Index decl_index, Index count, Type type) {
   }
 
   module_->features_used.simd |= (type == Type::V128);
+  module_->features_used.exceptions |= (type == Type::ExnRef);
   return Result::Ok;
 }
 
@@ -788,6 +818,7 @@ Result BinaryReaderIR::OnOpcode(Opcode opcode) {
     return AppendExpr(std::move(metadata));
   }
   module_->features_used.simd |= (opcode.GetResultType() == Type::V128);
+  module_->features_used.threads |= (opcode.GetPrefix() == 0xfe);
   return Result::Ok;
 }
 
@@ -796,7 +827,7 @@ Result BinaryReaderIR::OnAtomicLoadExpr(Opcode opcode,
                                         Address alignment_log2,
                                         Address offset) {
   return AppendExpr(std::make_unique<AtomicLoadExpr>(
-      opcode, Var(memidx, GetLocation()), 1 << alignment_log2, offset));
+      opcode, Var(memidx, GetLocation()), 1ull << alignment_log2, offset));
 }
 
 Result BinaryReaderIR::OnAtomicStoreExpr(Opcode opcode,
@@ -804,7 +835,7 @@ Result BinaryReaderIR::OnAtomicStoreExpr(Opcode opcode,
                                          Address alignment_log2,
                                          Address offset) {
   return AppendExpr(std::make_unique<AtomicStoreExpr>(
-      opcode, Var(memidx, GetLocation()), 1 << alignment_log2, offset));
+      opcode, Var(memidx, GetLocation()), 1ull << alignment_log2, offset));
 }
 
 Result BinaryReaderIR::OnAtomicRmwExpr(Opcode opcode,
@@ -812,7 +843,7 @@ Result BinaryReaderIR::OnAtomicRmwExpr(Opcode opcode,
                                        Address alignment_log2,
                                        Address offset) {
   return AppendExpr(std::make_unique<AtomicRmwExpr>(
-      opcode, Var(memidx, GetLocation()), 1 << alignment_log2, offset));
+      opcode, Var(memidx, GetLocation()), 1ull << alignment_log2, offset));
 }
 
 Result BinaryReaderIR::OnAtomicRmwCmpxchgExpr(Opcode opcode,
@@ -820,7 +851,7 @@ Result BinaryReaderIR::OnAtomicRmwCmpxchgExpr(Opcode opcode,
                                               Address alignment_log2,
                                               Address offset) {
   return AppendExpr(std::make_unique<AtomicRmwCmpxchgExpr>(
-      opcode, Var(memidx, GetLocation()), 1 << alignment_log2, offset));
+      opcode, Var(memidx, GetLocation()), 1ull << alignment_log2, offset));
 }
 
 Result BinaryReaderIR::OnAtomicWaitExpr(Opcode opcode,
@@ -828,7 +859,7 @@ Result BinaryReaderIR::OnAtomicWaitExpr(Opcode opcode,
                                         Address alignment_log2,
                                         Address offset) {
   return AppendExpr(std::make_unique<AtomicWaitExpr>(
-      opcode, Var(memidx, GetLocation()), 1 << alignment_log2, offset));
+      opcode, Var(memidx, GetLocation()), 1ull << alignment_log2, offset));
 }
 
 Result BinaryReaderIR::OnAtomicFenceExpr(uint32_t consistency_model) {
@@ -840,7 +871,7 @@ Result BinaryReaderIR::OnAtomicNotifyExpr(Opcode opcode,
                                           Address alignment_log2,
                                           Address offset) {
   return AppendExpr(std::make_unique<AtomicNotifyExpr>(
-      opcode, Var(memidx, GetLocation()), 1 << alignment_log2, offset));
+      opcode, Var(memidx, GetLocation()), 1ull << alignment_log2, offset));
 }
 
 Result BinaryReaderIR::OnBinaryExpr(Opcode opcode) {
@@ -891,15 +922,29 @@ Result BinaryReaderIR::OnCallRefExpr() {
 }
 
 Result BinaryReaderIR::OnReturnCallExpr(Index func_index) {
+  if (current_func_) {
+    // syntactically, a return_call expr can occur in an init expression
+    // (outside a function)
+    current_func_->features_used.tailcall = true;
+  }
   return AppendExpr(
       std::make_unique<ReturnCallExpr>(Var(func_index, GetLocation())));
 }
 
 Result BinaryReaderIR::OnReturnCallIndirectExpr(Index sig_index,
                                                 Index table_index) {
+  if (current_func_) {
+    // syntactically, a return_call_indirect expr can occur in an init
+    // expression (outside a function)
+    current_func_->features_used.tailcall = true;
+  }
   auto expr = std::make_unique<ReturnCallIndirectExpr>();
   SetFuncDeclaration(&expr->decl, Var(sig_index, GetLocation()));
   expr->table = Var(table_index, GetLocation());
+  FuncType* type = module_->GetFuncType(Var(sig_index, GetLocation()));
+  if (type) {
+    type->features_used.tailcall = true;
+  }
   return AppendExpr(std::move(expr));
 }
 
@@ -1013,7 +1058,7 @@ Result BinaryReaderIR::OnLoadExpr(Opcode opcode,
                                   Address alignment_log2,
                                   Address offset) {
   return AppendExpr(std::make_unique<LoadExpr>(
-      opcode, Var(memidx, GetLocation()), 1 << alignment_log2, offset));
+      opcode, Var(memidx, GetLocation()), 1ull << alignment_log2, offset));
 }
 
 Result BinaryReaderIR::OnLoopExpr(Type sig_type) {
@@ -1024,9 +1069,9 @@ Result BinaryReaderIR::OnLoopExpr(Type sig_type) {
   return PushLabel(LabelType::Loop, expr_list);
 }
 
-Result BinaryReaderIR::OnMemoryCopyExpr(Index srcmemidx, Index destmemidx) {
+Result BinaryReaderIR::OnMemoryCopyExpr(Index destmemidx, Index srcmemidx) {
   return AppendExpr(std::make_unique<MemoryCopyExpr>(
-      Var(srcmemidx, GetLocation()), Var(destmemidx, GetLocation())));
+      Var(destmemidx, GetLocation()), Var(srcmemidx, GetLocation())));
 }
 
 Result BinaryReaderIR::OnDataDropExpr(Index segment) {
@@ -1095,11 +1140,13 @@ Result BinaryReaderIR::OnTableFillExpr(Index table_index) {
 }
 
 Result BinaryReaderIR::OnRefFuncExpr(Index func_index) {
+  module_->used_func_refs.insert(func_index);
   return AppendExpr(
       std::make_unique<RefFuncExpr>(Var(func_index, GetLocation())));
 }
 
 Result BinaryReaderIR::OnRefNullExpr(Type type) {
+  module_->features_used.exceptions |= (type == Type::ExnRef);
   return AppendExpr(std::make_unique<RefNullExpr>(type));
 }
 
@@ -1140,10 +1187,11 @@ Result BinaryReaderIR::OnStoreExpr(Opcode opcode,
                                    Address alignment_log2,
                                    Address offset) {
   return AppendExpr(std::make_unique<StoreExpr>(
-      opcode, Var(memidx, GetLocation()), 1 << alignment_log2, offset));
+      opcode, Var(memidx, GetLocation()), 1ull << alignment_log2, offset));
 }
 
 Result BinaryReaderIR::OnThrowExpr(Index tag_index) {
+  module_->features_used.exceptions = true;
   return AppendExpr(std::make_unique<ThrowExpr>(Var(tag_index, GetLocation())));
 }
 
@@ -1255,7 +1303,8 @@ Result BinaryReaderIR::OnSimdLoadLaneExpr(Opcode opcode,
                                           Address offset,
                                           uint64_t value) {
   return AppendExpr(std::make_unique<SimdLoadLaneExpr>(
-      opcode, Var(memidx, GetLocation()), 1 << alignment_log2, offset, value));
+      opcode, Var(memidx, GetLocation()), 1ull << alignment_log2, offset,
+      value));
 }
 
 Result BinaryReaderIR::OnSimdStoreLaneExpr(Opcode opcode,
@@ -1264,7 +1313,8 @@ Result BinaryReaderIR::OnSimdStoreLaneExpr(Opcode opcode,
                                            Address offset,
                                            uint64_t value) {
   return AppendExpr(std::make_unique<SimdStoreLaneExpr>(
-      opcode, Var(memidx, GetLocation()), 1 << alignment_log2, offset, value));
+      opcode, Var(memidx, GetLocation()), 1ull << alignment_log2, offset,
+      value));
 }
 
 Result BinaryReaderIR::OnSimdShuffleOpExpr(Opcode opcode, v128 value) {
@@ -1276,7 +1326,7 @@ Result BinaryReaderIR::OnLoadSplatExpr(Opcode opcode,
                                        Address alignment_log2,
                                        Address offset) {
   return AppendExpr(std::make_unique<LoadSplatExpr>(
-      opcode, Var(memidx, GetLocation()), 1 << alignment_log2, offset));
+      opcode, Var(memidx, GetLocation()), 1ull << alignment_log2, offset));
 }
 
 Result BinaryReaderIR::OnLoadZeroExpr(Opcode opcode,
@@ -1284,7 +1334,7 @@ Result BinaryReaderIR::OnLoadZeroExpr(Opcode opcode,
                                       Address alignment_log2,
                                       Address offset) {
   return AppendExpr(std::make_unique<LoadZeroExpr>(
-      opcode, Var(memidx, GetLocation()), 1 << alignment_log2, offset));
+      opcode, Var(memidx, GetLocation()), 1ull << alignment_log2, offset));
 }
 
 Result BinaryReaderIR::OnElemSegmentCount(Index count) {
@@ -1349,26 +1399,16 @@ Result BinaryReaderIR::OnElemSegmentElemExprCount(Index index, Index count) {
   return Result::Ok;
 }
 
-Result BinaryReaderIR::OnElemSegmentElemExpr_RefNull(Index segment_index,
+Result BinaryReaderIR::BeginElemExpr(Index elem_index, Index expr_index) {
-                                                     Type type) {
+  assert(elem_index == module_->elem_segments.size() - 1);
-  assert(segment_index == module_->elem_segments.size() - 1);
+  ElemSegment* segment = module_->elem_segments[elem_index];
-  ElemSegment* segment = module_->elem_segments[segment_index];
+  assert(expr_index == segment->elem_exprs.size());
-  Location loc = GetLocation();
+  segment->elem_exprs.emplace_back();
-  ExprList init_expr;
+  return BeginInitExpr(&segment->elem_exprs.back());
-  init_expr.push_back(std::make_unique<RefNullExpr>(type, loc));
-  segment->elem_exprs.push_back(std::move(init_expr));
-  return Result::Ok;
 }
 
-Result BinaryReaderIR::OnElemSegmentElemExpr_RefFunc(Index segment_index,
+Result BinaryReaderIR::EndElemExpr(Index elem_index, Index expr_index) {
-                                                     Index func_index) {
+  return EndInitExpr();
-  assert(segment_index == module_->elem_segments.size() - 1);
-  ElemSegment* segment = module_->elem_segments[segment_index];
-  Location loc = GetLocation();
-  ExprList init_expr;
-  init_expr.push_back(std::make_unique<RefFuncExpr>(Var(func_index, loc), loc));
-  segment->elem_exprs.push_back(std::move(init_expr));
-  return Result::Ok;
 }
 
 Result BinaryReaderIR::OnDataSegmentCount(Index count) {
@@ -1580,6 +1620,7 @@ Result BinaryReaderIR::OnNameEntry(NameSectionSubsection type,
     case NameSectionSubsection::Local:
     case NameSectionSubsection::Module:
     case NameSectionSubsection::Label:
+    case NameSectionSubsection::Field:
       break;
     case NameSectionSubsection::Type:
       SetTypeName(index, name);
@@ -1760,6 +1801,18 @@ Result BinaryReaderIR::OnTableSymbol(Index index,
   return SetTableName(table_index, name);
 }
 
+Result BinaryReaderIR::OnGenericCustomSection(std::string_view name,
+                                              const void* data,
+                                              Offset size) {
+  Custom custom = Custom(GetLocation(), name);
+  custom.data.resize(size);
+  if (size > 0) {
+    memcpy(custom.data.data(), data, size);
+  }
+  module_->customs.push_back(std::move(custom));
+  return Result::Ok;
+}
+
 }  // end anonymous namespace
 
 Result ReadBinaryIr(const char* filename,

third_party/wasm2c/src/binary-reader-logging.cc

@@ -217,14 +217,15 @@ Result BinaryReaderLogging::OnImportMemory(Index import_index,
                                            std::string_view module_name,
                                            std::string_view field_name,
                                            Index memory_index,
-                                           const Limits* page_limits) {
+                                           const Limits* page_limits,
+                                           uint32_t page_size) {
   char buf[100];
   SPrintLimits(buf, sizeof(buf), page_limits);
   LOGF("OnImportMemory(import_index: %" PRIindex ", memory_index: %" PRIindex
        ", %s)\n",
        import_index, memory_index, buf);
   return reader_->OnImportMemory(import_index, module_name, field_name,
-                                 memory_index, page_limits);
+                                 memory_index, page_limits, page_size);
 }
 
 Result BinaryReaderLogging::OnImportGlobal(Index import_index,
@@ -264,11 +265,13 @@ Result BinaryReaderLogging::OnTable(Index index,
   return reader_->OnTable(index, elem_type, elem_limits);
 }
 
-Result BinaryReaderLogging::OnMemory(Index index, const Limits* page_limits) {
+Result BinaryReaderLogging::OnMemory(Index index,
+                                     const Limits* page_limits,
+                                     uint32_t page_size) {
   char buf[100];
   SPrintLimits(buf, sizeof(buf), page_limits);
   LOGF("OnMemory(index: %" PRIindex ", %s)\n", index, buf);
-  return reader_->OnMemory(index, page_limits);
+  return reader_->OnMemory(index, page_limits, page_size);
 }
 
 Result BinaryReaderLogging::BeginGlobal(Index index, Type type, bool mutable_) {
@@ -654,6 +657,15 @@ Result BinaryReaderLogging::OnCodeMetadata(Offset code_offset,
   return reader_->OnCodeMetadata(code_offset, data, size);
 }
 
+Result BinaryReaderLogging::OnGenericCustomSection(std::string_view name,
+                                                   const void* data,
+                                                   Offset size) {
+  LOGF("OnGenericCustomSection(name: \"" PRIstringview "\", size: %" PRIzd
+       ")\n",
+       WABT_PRINTF_STRING_VIEW_ARG(name), size);
+  return reader_->OnGenericCustomSection(name, data, size);
+}
+
 #define DEFINE_BEGIN(name)                        \
   Result BinaryReaderLogging::name(Offset size) { \
     LOGF(#name "(%" PRIzd ")\n", size);           \
@@ -786,6 +798,7 @@ DEFINE_BEGIN(BeginCodeSection)
 DEFINE_INDEX(OnFunctionBodyCount)
 DEFINE_INDEX(EndFunctionBody)
 DEFINE_INDEX(OnLocalDeclCount)
+DEFINE0(EndLocalDecls)
 DEFINE_LOAD_STORE_OPCODE(OnAtomicLoadExpr);
 DEFINE_LOAD_STORE_OPCODE(OnAtomicRmwExpr);
 DEFINE_LOAD_STORE_OPCODE(OnAtomicRmwCmpxchgExpr);
@@ -811,7 +824,7 @@ DEFINE_LOAD_STORE_OPCODE(OnLoadExpr);
 DEFINE_INDEX_DESC(OnLocalGetExpr, "index")
 DEFINE_INDEX_DESC(OnLocalSetExpr, "index")
 DEFINE_INDEX_DESC(OnLocalTeeExpr, "index")
-DEFINE_INDEX_INDEX(OnMemoryCopyExpr, "src_memory_index", "dest_memory_index")
+DEFINE_INDEX_INDEX(OnMemoryCopyExpr, "dest_memory_index", "src_memory_index")
 DEFINE_INDEX(OnDataDropExpr)
 DEFINE_INDEX(OnMemoryFillExpr)
 DEFINE_INDEX(OnMemoryGrowExpr)
@@ -850,8 +863,8 @@ DEFINE_INDEX(OnElemSegmentCount)
 DEFINE_INDEX(BeginElemSegmentInitExpr)
 DEFINE_INDEX(EndElemSegmentInitExpr)
 DEFINE_INDEX_INDEX(OnElemSegmentElemExprCount, "index", "count")
-DEFINE_INDEX_TYPE(OnElemSegmentElemExpr_RefNull)
+DEFINE_INDEX_INDEX(BeginElemExpr, "elem_index", "expr_index")
-DEFINE_INDEX_INDEX(OnElemSegmentElemExpr_RefFunc, "index", "func_index")
+DEFINE_INDEX_INDEX(EndElemExpr, "elem_index", "expr_index")
 DEFINE_INDEX(EndElemSegment)
 DEFINE_END(EndElemSection)
 
@@ -894,6 +907,9 @@ DEFINE_INDEX(OnInitFunctionCount)
 DEFINE_INDEX(OnComdatCount)
 DEFINE_END(EndLinkingSection)
 
+DEFINE_BEGIN(BeginGenericCustomSection);
+DEFINE_END(EndGenericCustomSection);
+
 DEFINE_BEGIN(BeginTagSection);
 DEFINE_INDEX(OnTagCount);
 DEFINE_INDEX_INDEX(OnTagType, "index", "sig_index")

third_party/wasm2c/src/binary-reader-objdump.cc

@@ -519,6 +519,8 @@ class BinaryReaderObjdumpDisassemble : public BinaryReaderObjdumpBase {
 
   std::string BlockSigToString(Type type) const;
 
+  Result OnFunction(Index index, Index sig_index) override;
+
   Result BeginFunctionBody(Index index, Offset size) override;
   Result EndFunctionBody(Index index) override;
 
@@ -633,13 +635,15 @@ Result BinaryReaderObjdumpDisassemble::OnLocalDecl(Index decl_index,
   for (size_t i = data_size; i < IMMEDIATE_OCTET_COUNT; i++) {
     printf("   ");
   }
-  printf(" | local[%" PRIindex, local_index_);
+  printf(" | local[");
+  if (count > 0) {
+    printf("%" PRIindex, local_index_);
 
-  if (count != 1) {
+    if (count != 1) {
-    printf("..%" PRIindex "", local_index_ + count - 1);
+      printf("..%" PRIindex "", local_index_ + count - 1);
+    }
+    local_index_ += count;
   }
-  local_index_ += count;
-
   printf("] type=%s\n", type.GetName().c_str());
 
   last_opcode_end = current_opcode_offset + data_size;
@@ -853,7 +857,7 @@ Result BinaryReaderObjdumpDisassemble::OnOpcodeUint64(uint64_t value) {
   if (!in_function_body) {
     return Result::Ok;
   }
-  LogOpcode("%" PRId64, value);
+  LogOpcode("%" PRIu64, value);
   return Result::Ok;
 }
 
@@ -940,6 +944,12 @@ Result BinaryReaderObjdumpDisassemble::OnEndExpr() {
   return Result::Ok;
 }
 
+Result BinaryReaderObjdumpDisassemble::OnFunction(Index index,
+                                                  Index sig_index) {
+  objdump_state_->function_types[index] = sig_index;
+  return Result::Ok;
+}
+
 Result BinaryReaderObjdumpDisassemble::BeginFunctionBody(Index index,
                                                          Offset size) {
   printf("%06" PRIzx " func[%" PRIindex "]", GetPrintOffset(state->offset),
@@ -953,7 +963,8 @@ Result BinaryReaderObjdumpDisassemble::BeginFunctionBody(Index index,
   last_opcode_end = 0;
   in_function_body = true;
   current_function_index = index;
-  local_index_ = objdump_state_->function_param_counts[index];
+  auto type_index = objdump_state_->function_types[index];
+  local_index_ = objdump_state_->function_param_counts[type_index];
   return Result::Ok;
 }
 
@@ -977,6 +988,7 @@ Result BinaryReaderObjdumpDisassemble::OnOpcodeBlockSig(Type sig_type) {
 }
 
 enum class InitExprType {
+  Invalid,
   I32,
   F32,
   I64,
@@ -1047,7 +1059,8 @@ class BinaryReaderObjdump : public BinaryReaderObjdumpBase {
                         std::string_view module_name,
                         std::string_view field_name,
                         Index memory_index,
-                        const Limits* page_limits) override;
+                        const Limits* page_limits,
+                        uint32_t page_size) override;
   Result OnImportGlobal(Index import_index,
                         std::string_view module_name,
                         std::string_view field_name,
@@ -1069,7 +1082,9 @@ class BinaryReaderObjdump : public BinaryReaderObjdumpBase {
                  const Limits* elem_limits) override;
 
   Result OnMemoryCount(Index count) override;
-  Result OnMemory(Index index, const Limits* limits) override;
+  Result OnMemory(Index index,
+                  const Limits* limits,
+                  uint32_t page_size) override;
 
   Result OnGlobalCount(Index count) override;
   Result BeginGlobal(Index index, Type type, bool mutable_) override;
@@ -1092,9 +1107,6 @@ class BinaryReaderObjdump : public BinaryReaderObjdumpBase {
                           uint8_t flags) override;
   Result OnElemSegmentElemType(Index index, Type elem_type) override;
   Result OnElemSegmentElemExprCount(Index index, Index count) override;
-  Result OnElemSegmentElemExpr_RefNull(Index segment_index, Type type) override;
-  Result OnElemSegmentElemExpr_RefFunc(Index segment_index,
-                                       Index func_index) override;
 
   void BeginInitExpr() { current_init_expr_.insts.clear(); }
 
@@ -1122,6 +1134,18 @@ class BinaryReaderObjdump : public BinaryReaderObjdumpBase {
 
   Result EndGlobalInitExpr(Index index) override { return EndInitExpr(); }
 
+  Result BeginElemExpr(Index elem_index, Index expr_index) override {
+    reading_elem_expr_ = true;
+    elem_index_ = expr_index;
+    BeginInitExpr();
+    return Result::Ok;
+  }
+
+  Result EndElemExpr(Index elem_index, Index expr_index) override {
+    assert(expr_index == elem_index_);
+    return EndInitExpr();
+  }
+
   Result OnDataSegmentCount(Index count) override;
   Result BeginDataSegment(Index index,
                           Index memory_index,
@@ -1208,6 +1232,8 @@ class BinaryReaderObjdump : public BinaryReaderObjdumpBase {
   Result OnI64ConstExpr(uint64_t value) override;
   Result OnF32ConstExpr(uint32_t value) override;
   Result OnF64ConstExpr(uint64_t value) override;
+  Result OnRefFuncExpr(Index func_index) override;
+  Result OnRefNullExpr(Type type) override;
   Result OnGlobalGetExpr(Index global_index) override;
   Result OnCodeMetadataCount(Index function_index, Index count) override;
   Result OnCodeMetadata(Offset code_offset,
@@ -1220,7 +1246,9 @@ class BinaryReaderObjdump : public BinaryReaderObjdumpBase {
   void PrintDetails(const char* fmt, ...);
   Result PrintSymbolFlags(uint32_t flags);
   Result PrintSegmentFlags(uint32_t flags);
-  void PrintInitExpr(const InitExpr& expr, bool as_unsigned = false);
+  void PrintInitExpr(const InitExpr& expr,
+                     bool as_unsigned = false,
+                     bool with_prefix = true);
   Result OnCount(Index count);
 
   std::unique_ptr<FileStream> out_stream_;
@@ -1230,7 +1258,8 @@ class BinaryReaderObjdump : public BinaryReaderObjdumpBase {
   bool reading_elem_init_expr_ = false;
   bool reading_data_init_expr_ = false;
   bool reading_global_init_expr_ = false;
-  InitExpr current_init_expr_;
+  bool reading_elem_expr_ = false;
+  InitExpr current_init_expr_{};
   uint8_t data_flags_ = 0;
   uint8_t elem_flags_ = 0;
   Index data_mem_index_ = 0;
@@ -1239,7 +1268,7 @@ class BinaryReaderObjdump : public BinaryReaderObjdumpBase {
 
   bool ReadingInitExpr() {
     return reading_elem_init_expr_ || reading_data_init_expr_ ||
-           reading_global_init_expr_;
+           reading_global_init_expr_ || reading_elem_expr_;
   }
 };
 
@@ -1452,6 +1481,7 @@ Result BinaryReaderObjdump::OnFunction(Index index, Index sig_index) {
     PrintDetails(" <" PRIstringview ">", WABT_PRINTF_STRING_VIEW_ARG(name));
   }
   PrintDetails("\n");
+  objdump_state_->function_types[index] = sig_index;
   return Result::Ok;
 }
 
@@ -1523,6 +1553,9 @@ Result BinaryReaderObjdump::OnImportTable(Index import_index,
   if (elem_limits->has_max) {
     PrintDetails(" max=%" PRId64, elem_limits->max);
   }
+  if (elem_limits->is_64) {
+    PrintDetails(" i64");
+  }
   PrintDetails(" <- " PRIstringview "." PRIstringview "\n",
                WABT_PRINTF_STRING_VIEW_ARG(module_name),
                WABT_PRINTF_STRING_VIEW_ARG(field_name));
@@ -1533,7 +1566,8 @@ Result BinaryReaderObjdump::OnImportMemory(Index import_index,
                                            std::string_view module_name,
                                            std::string_view field_name,
                                            Index memory_index,
-                                           const Limits* page_limits) {
+                                           const Limits* page_limits,
+                                           uint32_t page_size) {
   PrintDetails(" - memory[%" PRIindex "] pages: initial=%" PRId64, memory_index,
                page_limits->initial);
   if (page_limits->has_max) {
@@ -1545,6 +1579,9 @@ Result BinaryReaderObjdump::OnImportMemory(Index import_index,
   if (page_limits->is_64) {
     PrintDetails(" i64");
   }
+  if (page_size != WABT_DEFAULT_PAGE_SIZE) {
+    PrintDetails(" (pagesize %u)", page_size);
+  }
   PrintDetails(" <- " PRIstringview "." PRIstringview "\n",
                WABT_PRINTF_STRING_VIEW_ARG(module_name),
                WABT_PRINTF_STRING_VIEW_ARG(field_name));
@@ -1585,7 +1622,9 @@ Result BinaryReaderObjdump::OnMemoryCount(Index count) {
   return OnCount(count);
 }
 
-Result BinaryReaderObjdump::OnMemory(Index index, const Limits* page_limits) {
+Result BinaryReaderObjdump::OnMemory(Index index,
+                                     const Limits* page_limits,
+                                     uint32_t page_size) {
   PrintDetails(" - memory[%" PRIindex "] pages: initial=%" PRId64, index,
                page_limits->initial);
   if (page_limits->has_max) {
@@ -1597,6 +1636,9 @@ Result BinaryReaderObjdump::OnMemory(Index index, const Limits* page_limits) {
   if (page_limits->is_64) {
     PrintDetails(" i64");
   }
+  if (page_size != WABT_DEFAULT_PAGE_SIZE) {
+    PrintDetails(" (pagesize %u)", page_size);
+  }
   PrintDetails("\n");
   return Result::Ok;
 }
@@ -1617,6 +1659,9 @@ Result BinaryReaderObjdump::OnTable(Index index,
   if (!name.empty()) {
     PrintDetails(" <" PRIstringview ">", WABT_PRINTF_STRING_VIEW_ARG(name));
   }
+  if (elem_limits->is_64) {
+    PrintDetails(" i64");
+  }
   PrintDetails("\n");
   return Result::Ok;
 }
@@ -1642,27 +1687,6 @@ Result BinaryReaderObjdump::OnExport(Index index,
   return Result::Ok;
 }
 
-Result BinaryReaderObjdump::OnElemSegmentElemExpr_RefNull(Index segment_index,
-                                                          Type type) {
-  PrintDetails("  - elem[%" PRIu64 "] = ref.null %s\n",
-               elem_offset_ + elem_index_, type.GetName().c_str());
-  elem_index_++;
-  return Result::Ok;
-}
-
-Result BinaryReaderObjdump::OnElemSegmentElemExpr_RefFunc(Index segment_index,
-                                                          Index func_index) {
-  PrintDetails("  - elem[%" PRIu64 "] = func[%" PRIindex "]",
-               elem_offset_ + elem_index_, func_index);
-  auto name = GetFunctionName(func_index);
-  if (!name.empty()) {
-    PrintDetails(" <" PRIstringview ">", WABT_PRINTF_STRING_VIEW_ARG(name));
-  }
-  PrintDetails("\n");
-  elem_index_++;
-  return Result::Ok;
-}
-
 Result BinaryReaderObjdump::OnElemSegmentCount(Index count) {
   return OnCount(count);
 }
@@ -1709,14 +1733,22 @@ Result BinaryReaderObjdump::BeginGlobal(Index index, Type type, bool mutable_) {
 }
 
 void BinaryReaderObjdump::PrintInitExpr(const InitExpr& expr,
-                                        bool as_unsigned) {
+                                        bool as_unsigned,
-  assert(expr.insts.size() > 0);
+                                        bool with_prefix) {
+  if (with_prefix) {
+    PrintDetails(" - init ");
+  }
+
+  if (expr.insts.empty()) {
+    PrintDetails("<EMPTY>\n");
+    return;
+  }
 
   // We have two different way to print init expressions.  One for
   // extended expressions involving more than one instruction, and
   // a short form for the more traditional single instruction form.
   if (expr.insts.size() > 1) {
-    PrintDetails(" - init (");
+    PrintDetails("(");
     bool first = true;
     for (auto& inst : expr.insts) {
       if (!first) {
@@ -1763,39 +1795,39 @@ void BinaryReaderObjdump::PrintInitExpr(const InitExpr& expr,
   switch (expr.type) {
     case InitExprType::I32:
       if (as_unsigned) {
-        PrintDetails(" - init i32=%u\n", expr.insts[0].imm.i32);
+        PrintDetails("i32=%u\n", expr.insts[0].imm.i32);
       } else {
-        PrintDetails(" - init i32=%d\n", expr.insts[0].imm.i32);
+        PrintDetails("i32=%d\n", expr.insts[0].imm.i32);
       }
       break;
     case InitExprType::I64:
       if (as_unsigned) {
-        PrintDetails(" - init i64=%" PRIu64 "\n", expr.insts[0].imm.i64);
+        PrintDetails("i64=%" PRIu64 "\n", expr.insts[0].imm.i64);
       } else {
-        PrintDetails(" - init i64=%" PRId64 "\n", expr.insts[0].imm.i64);
+        PrintDetails("i64=%" PRId64 "\n", expr.insts[0].imm.i64);
       }
       break;
     case InitExprType::F64: {
       char buffer[WABT_MAX_DOUBLE_HEX];
       WriteDoubleHex(buffer, sizeof(buffer), expr.insts[0].imm.f64);
-      PrintDetails(" - init f64=%s\n", buffer);
+      PrintDetails("f64=%s\n", buffer);
       break;
     }
     case InitExprType::F32: {
       char buffer[WABT_MAX_FLOAT_HEX];
       WriteFloatHex(buffer, sizeof(buffer), expr.insts[0].imm.f32);
-      PrintDetails(" - init f32=%s\n", buffer);
+      PrintDetails("f32=%s\n", buffer);
       break;
     }
     case InitExprType::V128: {
       PrintDetails(
-          " - init v128=0x%08x 0x%08x 0x%08x 0x%08x \n",
+          "v128=0x%08x 0x%08x 0x%08x 0x%08x \n",
           expr.insts[0].imm.v128_v.u32(0), expr.insts[0].imm.v128_v.u32(1),
           expr.insts[0].imm.v128_v.u32(2), expr.insts[0].imm.v128_v.u32(3));
       break;
     }
     case InitExprType::Global: {
-      PrintDetails(" - init global=%" PRIindex, expr.insts[0].imm.index);
+      PrintDetails("global=%" PRIindex, expr.insts[0].imm.index);
       std::string_view name = GetGlobalName(expr.insts[0].imm.index);
       if (!name.empty()) {
         PrintDetails(" <" PRIstringview ">", WABT_PRINTF_STRING_VIEW_ARG(name));
@@ -1804,7 +1836,7 @@ void BinaryReaderObjdump::PrintInitExpr(const InitExpr& expr,
       break;
     }
     case InitExprType::FuncRef: {
-      PrintDetails(" - init ref.func:%" PRIindex, expr.insts[0].imm.index);
+      PrintDetails("ref.func:%" PRIindex, expr.insts[0].imm.index);
       std::string_view name = GetFunctionName(expr.insts[0].imm.index);
       if (!name.empty()) {
         PrintDetails(" <" PRIstringview ">", WABT_PRINTF_STRING_VIEW_ARG(name));
@@ -1813,8 +1845,10 @@ void BinaryReaderObjdump::PrintInitExpr(const InitExpr& expr,
       break;
     }
     case InitExprType::NullRef:
-      PrintDetails(" - init null\n");
+      PrintDetails("ref.null %s\n", expr.insts[0].imm.type.GetName().c_str());
       break;
+    case InitExprType::Invalid:
+      PrintDetails("<INVALID>\n");
       break;
   }
 }
@@ -1844,6 +1878,11 @@ Result BinaryReaderObjdump::EndInitExpr() {
   } else if (reading_global_init_expr_) {
     reading_global_init_expr_ = false;
     PrintInitExpr(current_init_expr_);
+  } else if (reading_elem_expr_) {
+    reading_elem_expr_ = false;
+    PrintDetails("  - elem[%" PRIu64 "] = ", elem_offset_ + elem_index_);
+    PrintInitExpr(current_init_expr_, /*as_unsigned=*/false,
+                  /*with_prefix=*/false);
   } else {
     WABT_UNREACHABLE;
   }
@@ -1882,6 +1921,22 @@ Result BinaryReaderObjdump::OnF64ConstExpr(uint64_t value) {
   return Result::Ok;
 }
 
+Result BinaryReaderObjdump::OnRefFuncExpr(Index func_index) {
+  if (ReadingInitExpr()) {
+    current_init_expr_.type = InitExprType::FuncRef;
+    current_init_expr_.insts.back().imm.index = func_index;
+  }
+  return Result::Ok;
+}
+
+Result BinaryReaderObjdump::OnRefNullExpr(Type type) {
+  if (ReadingInitExpr()) {
+    current_init_expr_.type = InitExprType::NullRef;
+    current_init_expr_.insts.back().imm.type = type;
+  }
+  return Result::Ok;
+}
+
 Result BinaryReaderObjdump::OnOpcode(Opcode opcode) {
   BinaryReaderObjdumpBase::OnOpcode(opcode);
   if (ReadingInitExpr() && opcode != Opcode::End) {
@@ -2144,6 +2199,10 @@ Result BinaryReaderObjdump::PrintSymbolFlags(uint32_t flags) {
     PrintDetails(" tls");
     flags &= ~WABT_SYMBOL_FLAG_TLS;
   }
+  if (flags & WABT_SYMBOL_FLAG_ABS) {
+    PrintDetails(" abs");
+    flags &= ~WABT_SYMBOL_FLAG_ABS;
+  }
   if (flags != 0) {
     PrintDetails(" unknown_flags=%#x", flags);
   }
@@ -2165,6 +2224,10 @@ Result BinaryReaderObjdump::PrintSegmentFlags(uint32_t flags) {
     PrintDetails(" TLS");
     flags &= ~WABT_SEGMENT_FLAG_TLS;
   }
+  if (flags & WASM_SEGMENT_FLAG_RETAIN) {
+    PrintDetails(" RETAIN");
+    flags &= ~WASM_SEGMENT_FLAG_RETAIN;
+  }
   if (flags != 0) {
     PrintDetails(" unknown_flags=%#x", flags);
   }
@@ -2180,9 +2243,14 @@ Result BinaryReaderObjdump::OnDataSymbol(Index index,
                                          uint32_t size) {
   PrintDetails("   - %d: D <" PRIstringview ">", index,
                WABT_PRINTF_STRING_VIEW_ARG(name));
-  if (!(flags & WABT_SYMBOL_FLAG_UNDEFINED))
+  if (!(flags & WABT_SYMBOL_FLAG_UNDEFINED)) {
-    PrintDetails(" segment=%" PRIindex " offset=%d size=%d", segment, offset,
+    if (flags & WABT_SYMBOL_FLAG_ABS) {
-                 size);
+      PrintDetails(" address=%d size=%d", offset, size);
+    } else {
+      PrintDetails(" segment=%" PRIindex " offset=%d size=%d", segment, offset,
+                   size);
+    }
+  }
   return PrintSymbolFlags(flags);
 }
 

third_party/wasm2c/src/binary-reader-stats.cc

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-#include "wabt/binary-reader-opcnt.h"
+#include "wabt/binary-reader-stats.h"
 
 #include <cassert>
 #include <cinttypes>

third_party/wasm2c/src/binary-reader.cc

@@ -22,6 +22,7 @@
 #include <cstdint>
 #include <cstdio>
 #include <cstring>
+#include <stack>
 #include <vector>
 
 #include "wabt/config.h"
@@ -115,9 +116,14 @@ class BinaryReader {
   [[nodiscard]] Result ReadBytes(const void** out_data,
                                  Address* out_data_size,
                                  const char* desc);
+  [[nodiscard]] Result ReadBytesWithSize(const void** out_data,
+                                         Offset size,
+                                         const char* desc);
   [[nodiscard]] Result ReadIndex(Index* index, const char* desc);
   [[nodiscard]] Result ReadOffset(Offset* offset, const char* desc);
   [[nodiscard]] Result ReadAlignment(Address* align_log2, const char* desc);
+  [[nodiscard]] Result CheckAlignment(Address* align_log2, const char* desc);
+  [[nodiscard]] Result TakeHasMemidx(Address* align_log2, bool* has_memidx);
   [[nodiscard]] Result ReadMemidx(Index* memidx, const char* desc);
   [[nodiscard]] Result ReadMemLocation(Address* alignment_log2,
                                        Index* memidx,
@@ -140,21 +146,21 @@ class BinaryReader {
 
   [[nodiscard]] Result ReadInitExpr(Index index);
   [[nodiscard]] Result ReadTable(Type* out_elem_type, Limits* out_elem_limits);
-  [[nodiscard]] Result ReadMemory(Limits* out_page_limits);
+  [[nodiscard]] Result ReadMemory(Limits* out_page_limits,
+                                  uint32_t* out_page_size);
   [[nodiscard]] Result ReadGlobalHeader(Type* out_type, bool* out_mutable);
   [[nodiscard]] Result ReadTagType(Index* out_sig_index);
   [[nodiscard]] Result ReadAddress(Address* out_value,
                                    Index memory,
                                    const char* desc);
   [[nodiscard]] Result ReadFunctionBody(Offset end_offset);
-  // ReadInstructions either until and END instruction, or until
+  // ReadInstructions reads until end_offset or the nesting depth reaches zero.
-  // the given end_offset.
+  [[nodiscard]] Result ReadInstructions(Offset end_offset, const char* context);
-  [[nodiscard]] Result ReadInstructions(bool stop_on_end,
-                                        Offset end_offset,
-                                        Opcode* final_opcode);
   [[nodiscard]] Result ReadNameSection(Offset section_size);
   [[nodiscard]] Result ReadRelocSection(Offset section_size);
   [[nodiscard]] Result ReadDylinkSection(Offset section_size);
+  [[nodiscard]] Result ReadGenericCustomSection(std::string_view name,
+                                                Offset section_size);
   [[nodiscard]] Result ReadDylink0Section(Offset section_size);
   [[nodiscard]] Result ReadTargetFeaturesSections(Offset section_size);
   [[nodiscard]] Result ReadLinkingSection(Offset section_size);
@@ -197,8 +203,8 @@ class BinaryReader {
   Index num_tag_imports_ = 0;
   Index num_function_signatures_ = 0;
   Index num_function_bodies_ = 0;
+  Index num_data_segments_ = 0;
   Index data_count_ = kInvalidIndex;
-  std::vector<Limits> memories;
 
   using ReadEndRestoreGuard =
       ValueRestoreGuard<size_t, &BinaryReader::read_end_>;
@@ -397,13 +403,19 @@ Result BinaryReader::ReadBytes(const void** out_data,
                                const char* desc) {
   uint32_t data_size = 0;
   CHECK_RESULT(ReadU32Leb128(&data_size, "data size"));
+  CHECK_RESULT(ReadBytesWithSize(out_data, data_size, desc));
+  *out_data_size = data_size;
+  return Result::Ok;
+}
 
-  ERROR_UNLESS(state_.offset + data_size <= read_end_,
+Result BinaryReader::ReadBytesWithSize(const void** out_data,
-               "unable to read data: %s", desc);
+                                       Offset size,
+                                       const char* desc) {
+  ERROR_UNLESS(state_.offset + size <= read_end_, "unable to read data: %s",
+               desc);
 
   *out_data = static_cast<const uint8_t*>(state_.data) + state_.offset;
-  *out_data_size = data_size;
+  state_.offset += size;
-  state_.offset += data_size;
   return Result::Ok;
 }
 
@@ -424,19 +436,29 @@ Result BinaryReader::ReadOffset(Offset* offset, const char* desc) {
 Result BinaryReader::ReadAlignment(Address* alignment_log2, const char* desc) {
   uint32_t value;
   CHECK_RESULT(ReadU32Leb128(&value, desc));
-  if (value >= 128 ||
+  *alignment_log2 = value;
-      (value >= 32 && !options_.features.multi_memory_enabled())) {
+  return Result::Ok;
-    PrintError("invalid %s: %u", desc, value);
+}
+
+Result BinaryReader::CheckAlignment(Address* align_log2, const char* desc) {
+  uint32_t value = *align_log2;
+  if (value >= 32) {
+    PrintError("invalid %s: %" PRIu32, desc, value);
     return Result::Error;
   }
-  *alignment_log2 = value;
+  return Result::Ok;
+}
+
+Result BinaryReader::TakeHasMemidx(Address* align_log2, bool* has_memidx) {
+  // extract the has_memidx flag
+  *has_memidx = (*align_log2 >> 6) & 1;
+  // then clear it
+  *align_log2 &= ~(1 << 6);
   return Result::Ok;
 }
 
 Result BinaryReader::ReadMemidx(Index* memidx, const char* desc) {
   CHECK_RESULT(ReadIndex(memidx, desc));
-  ERROR_UNLESS(*memidx < memories.size(), "memory index %u out of range",
-               *memidx);
   return Result::Ok;
 }
 
@@ -447,12 +469,14 @@ Result BinaryReader::ReadMemLocation(Address* alignment_log2,
                                      const char* desc_memidx,
                                      const char* desc_offset,
                                      uint8_t* lane_val) {
+  bool has_memidx = false;
   CHECK_RESULT(ReadAlignment(alignment_log2, desc_align));
+  CHECK_RESULT(TakeHasMemidx(alignment_log2, &has_memidx));
+  CHECK_RESULT(CheckAlignment(alignment_log2, desc_align));
   *memidx = 0;
-  if (*alignment_log2 >> 6) {
+  if (has_memidx) {
     ERROR_IF(!options_.features.multi_memory_enabled(),
              "multi_memory not allowed");
-    *alignment_log2 = *alignment_log2 & ((1 << 6) - 1);
     CHECK_RESULT(ReadMemidx(memidx, desc_memidx));
   }
   CHECK_RESULT(ReadAddress(offset, 0, desc_offset));
@@ -534,6 +558,9 @@ bool BinaryReader::IsConcreteType(Type type) {
     case Type::ExternRef:
       return options_.features.reference_types_enabled();
 
+    case Type::ExnRef:
+      return options_.features.exceptions_enabled();
+
     case Type::Reference:
       return options_.features.function_references_enabled();
 
@@ -559,14 +586,8 @@ Index BinaryReader::NumTotalFuncs() {
 }
 
 Result BinaryReader::ReadInitExpr(Index index) {
-  // Read instructions until END opcode is reached.
+  CHECK_RESULT(ReadInstructions(read_end_, "init expression"));
-  Opcode final_opcode(Opcode::Invalid);
+  assert(state_.offset <= read_end_);
-  CHECK_RESULT(
-      ReadInstructions(/*stop_on_end=*/true, read_end_, &final_opcode));
-  ERROR_UNLESS(state_.offset <= read_end_,
-               "init expression longer than given size");
-  ERROR_UNLESS(final_opcode == Opcode::End,
-               "init expression must end with END opcode");
   return Result::Ok;
 }
 
@@ -580,9 +601,10 @@ Result BinaryReader::ReadTable(Type* out_elem_type, Limits* out_elem_limits) {
   bool has_max = flags & WABT_BINARY_LIMITS_HAS_MAX_FLAG;
   bool is_shared = flags & WABT_BINARY_LIMITS_IS_SHARED_FLAG;
   bool is_64 = flags & WABT_BINARY_LIMITS_IS_64_FLAG;
-  const uint8_t unknown_flags = flags & ~WABT_BINARY_LIMITS_ALL_FLAGS;
+  const uint8_t unknown_flags = flags & ~WABT_BINARY_LIMITS_ALL_TABLE_FLAGS;
   ERROR_IF(is_shared, "tables may not be shared");
-  ERROR_IF(is_64, "tables may not be 64-bit");
+  ERROR_IF(is_64 && !options_.features.memory64_enabled(),
+           "memory64 not allowed");
   ERROR_UNLESS(unknown_flags == 0, "malformed table limits flag: %d", flags);
   CHECK_RESULT(ReadU32Leb128(&initial, "table initial elem count"));
   if (has_max) {
@@ -590,12 +612,14 @@ Result BinaryReader::ReadTable(Type* out_elem_type, Limits* out_elem_limits) {
   }
 
   out_elem_limits->has_max = has_max;
+  out_elem_limits->is_64 = is_64;
   out_elem_limits->initial = initial;
   out_elem_limits->max = max;
   return Result::Ok;
 }
 
-Result BinaryReader::ReadMemory(Limits* out_page_limits) {
+Result BinaryReader::ReadMemory(Limits* out_page_limits,
+                                uint32_t* out_page_size) {
   uint8_t flags;
   uint64_t initial;
   uint64_t max = 0;
@@ -603,13 +627,18 @@ Result BinaryReader::ReadMemory(Limits* out_page_limits) {
   bool has_max = flags & WABT_BINARY_LIMITS_HAS_MAX_FLAG;
   bool is_shared = flags & WABT_BINARY_LIMITS_IS_SHARED_FLAG;
   bool is_64 = flags & WABT_BINARY_LIMITS_IS_64_FLAG;
-  const uint8_t unknown_flags = flags & ~WABT_BINARY_LIMITS_ALL_FLAGS;
+  bool has_custom_page_size =
+      flags & WABT_BINARY_LIMITS_HAS_CUSTOM_PAGE_SIZE_FLAG;
+  const uint8_t unknown_flags = flags & ~WABT_BINARY_LIMITS_ALL_MEMORY_FLAGS;
   ERROR_UNLESS(unknown_flags == 0, "malformed memory limits flag: %d", flags);
   ERROR_IF(is_shared && !options_.features.threads_enabled(),
            "memory may not be shared: threads not allowed");
   ERROR_IF(is_64 && !options_.features.memory64_enabled(),
            "memory64 not allowed");
-  if (is_64) {
+  ERROR_IF(
+      has_custom_page_size && !options_.features.custom_page_sizes_enabled(),
+      "custom page sizes not allowed");
+  if (options_.features.memory64_enabled()) {
     CHECK_RESULT(ReadU64Leb128(&initial, "memory initial page count"));
     if (has_max) {
       CHECK_RESULT(ReadU64Leb128(&max, "memory max page count"));
@@ -624,6 +653,14 @@ Result BinaryReader::ReadMemory(Limits* out_page_limits) {
       max = max32;
     }
   }
+  if (has_custom_page_size) {
+    uint32_t page_size_log2;
+    CHECK_RESULT(ReadU32Leb128(&page_size_log2, "memory page size"));
+    ERROR_IF(page_size_log2 > 16, "malformed memory page size");
+    *out_page_size = 1 << page_size_log2;
+  } else {
+    *out_page_size = WABT_DEFAULT_PAGE_SIZE;
+  }
 
   out_page_limits->has_max = has_max;
   out_page_limits->is_shared = is_shared;
@@ -631,8 +668,6 @@ Result BinaryReader::ReadMemory(Limits* out_page_limits) {
   out_page_limits->initial = initial;
   out_page_limits->max = max;
 
-  // Have to keep a copy of these, to know how to interpret load/stores.
-  memories.push_back(*out_page_limits);
   return Result::Ok;
 }
 
@@ -654,10 +689,7 @@ Result BinaryReader::ReadGlobalHeader(Type* out_type, bool* out_mutable) {
 Result BinaryReader::ReadAddress(Address* out_value,
                                  Index memory,
                                  const char* desc) {
-  ERROR_UNLESS(memory < memories.size(),
+  if (options_.features.memory64_enabled()) {
-               "load/store memory %u out of range %zu", memory,
-               memories.size());
-  if (memories[memory].is_64) {
     return ReadU64Leb128(out_value, desc);
   } else {
     uint32_t val;
@@ -668,27 +700,19 @@ Result BinaryReader::ReadAddress(Address* out_value,
 }
 
 Result BinaryReader::ReadFunctionBody(Offset end_offset) {
-  Opcode final_opcode(Opcode::Invalid);
+  CHECK_RESULT(ReadInstructions(end_offset, "function body"));
-  CHECK_RESULT(
-      ReadInstructions(/*stop_on_end=*/false, end_offset, &final_opcode));
   ERROR_UNLESS(state_.offset == end_offset,
-               "function body longer than given size");
+               "function body shorter than given size");
-  ERROR_UNLESS(final_opcode == Opcode::End,
-               "function body must end with END opcode");
   return Result::Ok;
 }
 
-Result BinaryReader::ReadInstructions(bool stop_on_end,
+Result BinaryReader::ReadInstructions(Offset end_offset, const char* context) {
-                                      Offset end_offset,
+  std::stack<Opcode> nested_blocks;
-                                      Opcode* final_opcode) {
   while (state_.offset < end_offset) {
     Opcode opcode;
     CHECK_RESULT(ReadOpcode(&opcode, "opcode"));
     CALLBACK(OnOpcode, opcode);
     ERROR_UNLESS_OPCODE_ENABLED(opcode);
-    if (final_opcode) {
-      *final_opcode = opcode;
-    }
 
     switch (opcode) {
       case Opcode::Unreachable:
@@ -697,6 +721,7 @@ Result BinaryReader::ReadInstructions(bool stop_on_end,
         break;
 
       case Opcode::Block: {
+        nested_blocks.push(opcode);
         Type sig_type;
         CHECK_RESULT(ReadType(&sig_type, "block signature type"));
         ERROR_UNLESS(IsBlockType(sig_type),
@@ -707,6 +732,7 @@ Result BinaryReader::ReadInstructions(bool stop_on_end,
       }
 
       case Opcode::Loop: {
+        nested_blocks.push(opcode);
         Type sig_type;
         CHECK_RESULT(ReadType(&sig_type, "loop signature type"));
         ERROR_UNLESS(IsBlockType(sig_type),
@@ -717,6 +743,7 @@ Result BinaryReader::ReadInstructions(bool stop_on_end,
       }
 
       case Opcode::If: {
+        nested_blocks.push(opcode);
         Type sig_type;
         CHECK_RESULT(ReadType(&sig_type, "if signature type"));
         ERROR_UNLESS(IsBlockType(sig_type),
@@ -727,6 +754,8 @@ Result BinaryReader::ReadInstructions(bool stop_on_end,
       }
 
       case Opcode::Else:
+        ERROR_IF(nested_blocks.empty() || (nested_blocks.top() != Opcode::If),
+                 "else outside if block");
         CALLBACK0(OnElseExpr);
         CALLBACK0(OnOpcodeBare);
         break;
@@ -816,9 +845,10 @@ Result BinaryReader::ReadInstructions(bool stop_on_end,
 
       case Opcode::End:
         CALLBACK0(OnEndExpr);
-        if (stop_on_end) {
+        if (nested_blocks.empty()) {
           return Result::Ok;
         }
+        nested_blocks.pop();
         break;
 
       case Opcode::I32Const: {
@@ -1484,6 +1514,7 @@ Result BinaryReader::ReadInstructions(bool stop_on_end,
         break;
 
       case Opcode::Try: {
+        nested_blocks.push(opcode);
         Type sig_type;
         CHECK_RESULT(ReadType(&sig_type, "try signature type"));
         ERROR_UNLESS(IsBlockType(sig_type),
@@ -1508,6 +1539,9 @@ Result BinaryReader::ReadInstructions(bool stop_on_end,
       }
 
       case Opcode::Delegate: {
+        ERROR_IF(nested_blocks.empty() || (nested_blocks.top() != Opcode::Try),
+                 "delegate outside try block");
+        nested_blocks.pop();
         Index index;
         CHECK_RESULT(ReadIndex(&index, "depth"));
         CALLBACK(OnDelegateExpr, index);
@@ -1754,8 +1788,8 @@ Result BinaryReader::ReadInstructions(bool stop_on_end,
       }
 
       case Opcode::MemoryCopy: {
-        Index srcmemidx = 0;
         Index destmemidx = 0;
+        Index srcmemidx = 0;
         if (!options_.features.multi_memory_enabled()) {
           uint8_t reserved;
           CHECK_RESULT(ReadU8(&reserved, "reserved memory index"));
@@ -1763,11 +1797,11 @@ Result BinaryReader::ReadInstructions(bool stop_on_end,
           CHECK_RESULT(ReadU8(&reserved, "reserved memory index"));
           ERROR_UNLESS(reserved == 0, "reserved value must be 0");
         } else {
-          CHECK_RESULT(ReadMemidx(&srcmemidx, "memory.copy srcmemidx"));
           CHECK_RESULT(ReadMemidx(&destmemidx, "memory.copy destmemindex"));
+          CHECK_RESULT(ReadMemidx(&srcmemidx, "memory.copy srcmemidx"));
         }
-        CALLBACK(OnMemoryCopyExpr, srcmemidx, destmemidx);
+        CALLBACK(OnMemoryCopyExpr, destmemidx, srcmemidx);
-        CALLBACK(OnOpcodeUint32Uint32, srcmemidx, destmemidx);
+        CALLBACK(OnOpcodeUint32Uint32, destmemidx, srcmemidx);
         break;
       }
 
@@ -1851,7 +1885,9 @@ Result BinaryReader::ReadInstructions(bool stop_on_end,
         return ReportUnexpectedOpcode(opcode);
     }
   }
-  return Result::Ok;
+
+  PrintError("%s must end with END opcode", context);
+  return Result::Error;
 }
 
 Result BinaryReader::ReadNameSection(Offset section_size) {
@@ -2019,14 +2055,17 @@ Result BinaryReader::ReadRelocSection(Offset section_size) {
       case RelocType::MemoryAddressRelSLEB64:
       case RelocType::MemoryAddressI32:
       case RelocType::MemoryAddressI64:
+      case RelocType::MemoryAddressLocRelI32:
       case RelocType::FunctionOffsetI32:
+      case RelocType::FunctionOffsetI64:
       case RelocType::SectionOffsetI32:
       case RelocType::MemoryAddressTLSSLEB:
-      case RelocType::MemoryAddressTLSI32:
+      case RelocType::MemoryAddressTLSSLEB64:
         CHECK_RESULT(ReadS32Leb128(&addend, "addend"));
         break;
 
       case RelocType::FuncIndexLEB:
+      case RelocType::FuncIndexI32:
       case RelocType::TableIndexSLEB:
       case RelocType::TableIndexSLEB64:
       case RelocType::TableIndexI32:
@@ -2036,6 +2075,7 @@ Result BinaryReader::ReadRelocSection(Offset section_size) {
       case RelocType::GlobalIndexI32:
       case RelocType::TagIndexLEB:
       case RelocType::TableIndexRelSLEB:
+      case RelocType::TableIndexRelSLEB64:
       case RelocType::TableNumberLEB:
         break;
 
@@ -2167,6 +2207,18 @@ Result BinaryReader::ReadTargetFeaturesSections(Offset section_size) {
   return Result::Ok;
 }
 
+Result BinaryReader::ReadGenericCustomSection(std::string_view name,
+                                              Offset section_size) {
+  CALLBACK(BeginGenericCustomSection, section_size);
+  const void* data;
+  Offset custom_data_size = read_end_ - state_.offset;
+  CHECK_RESULT(
+      ReadBytesWithSize(&data, custom_data_size, "custom section data"));
+  CALLBACK(OnGenericCustomSection, name, data, custom_data_size);
+  CALLBACK0(EndGenericCustomSection);
+  return Result::Ok;
+}
+
 Result BinaryReader::ReadLinkingSection(Offset section_size) {
   CALLBACK(BeginLinkingSection, section_size);
   uint32_t version;
@@ -2254,6 +2306,7 @@ Result BinaryReader::ReadLinkingSection(Offset section_size) {
           uint32_t flags;
           CHECK_RESULT(ReadStr(&name, "segment name"));
           CHECK_RESULT(ReadAlignment(&alignment_log2, "segment alignment"));
+          CHECK_RESULT(CheckAlignment(&alignment_log2, "segment alignment"));
           CHECK_RESULT(ReadU32Leb128(&flags, "segment flags"));
           CALLBACK(OnSegmentInfo, i, name, alignment_log2, flags);
         }
@@ -2387,6 +2440,13 @@ Result BinaryReader::ReadCustomSection(Index section_index,
   ValueRestoreGuard<bool, &BinaryReader::reading_custom_section_> guard(this);
   reading_custom_section_ = true;
 
+  {
+    // Backtrack parser when scope ends
+    ValueRestoreGuard<BinaryReaderDelegate::State, &BinaryReader::state_> guard(
+        this);
+    CHECK_RESULT(ReadGenericCustomSection(section_name, section_size));
+  }
+
   if (options_.read_debug_names && section_name == WABT_BINARY_SECTION_NAME) {
     CHECK_RESULT(ReadNameSection(section_size));
     did_read_names_section_ = true;
@@ -2407,7 +2467,7 @@ Result BinaryReader::ReadCustomSection(Index section_index,
     metadata_name.remove_prefix(sizeof(WABT_BINARY_SECTION_CODE_METADATA) - 1);
     CHECK_RESULT(ReadCodeMetadataSection(metadata_name, section_size));
   } else {
-    // This is an unknown custom section, skip it.
+    // Skip. This is a generic custom section, and is handled above.
     state_.offset = read_end_;
   }
   CALLBACK0(EndCustomSection);
@@ -2541,9 +2601,10 @@ Result BinaryReader::ReadImportSection(Offset section_size) {
 
       case ExternalKind::Memory: {
         Limits page_limits;
-        CHECK_RESULT(ReadMemory(&page_limits));
+        uint32_t page_size;
+        CHECK_RESULT(ReadMemory(&page_limits, &page_size));
         CALLBACK(OnImportMemory, i, module_name, field_name,
-                 num_memory_imports_, &page_limits);
+                 num_memory_imports_, &page_limits, page_size);
         num_memory_imports_++;
         break;
       }
@@ -2618,8 +2679,9 @@ Result BinaryReader::ReadMemorySection(Offset section_size) {
   for (Index i = 0; i < num_memories; ++i) {
     Index memory_index = num_memory_imports_ + i;
     Limits page_limits;
-    CHECK_RESULT(ReadMemory(&page_limits));
+    uint32_t page_size;
-    CALLBACK(OnMemory, memory_index, &page_limits);
+    CHECK_RESULT(ReadMemory(&page_limits, &page_size));
+    CALLBACK(OnMemory, memory_index, &page_limits, page_size);
   }
   CALLBACK0(EndMemorySection);
   return Result::Ok;
@@ -2723,29 +2785,18 @@ Result BinaryReader::ReadElemSection(Offset section_size) {
 
     CALLBACK(OnElemSegmentElemExprCount, i, num_elem_exprs);
     for (Index j = 0; j < num_elem_exprs; ++j) {
+      CALLBACK(BeginElemExpr, i, j);
       if (flags & SegUseElemExprs) {
-        Opcode opcode;
+        CHECK_RESULT(ReadInitExpr(j));
-        CHECK_RESULT(ReadOpcode(&opcode, "elem expr opcode"));
-        if (opcode == Opcode::RefNull) {
-          Type type;
-          CHECK_RESULT(ReadRefType(&type, "elem expr ref.null type"));
-          CALLBACK(OnElemSegmentElemExpr_RefNull, i, type);
-        } else if (opcode == Opcode::RefFunc) {
-          Index func_index;
-          CHECK_RESULT(ReadIndex(&func_index, "elem expr func index"));
-          CALLBACK(OnElemSegmentElemExpr_RefFunc, i, func_index);
-        } else {
-          PrintError(
-              "expected ref.null or ref.func in passive element segment");
-        }
-        CHECK_RESULT(ReadOpcode(&opcode, "opcode"));
-        ERROR_UNLESS(opcode == Opcode::End,
-                     "expected END opcode after element expression");
       } else {
         Index func_index;
         CHECK_RESULT(ReadIndex(&func_index, "elem expr func index"));
-        CALLBACK(OnElemSegmentElemExpr_RefFunc, i, func_index);
+        CALLBACK(OnOpcode, Opcode::RefFunc);
+        CALLBACK(OnRefFuncExpr, func_index);
+        CALLBACK(OnOpcodeUint32, func_index);
+        CALLBACK0(OnEndExpr);
       }
+      CALLBACK(EndElemExpr, i, j);
     }
     CALLBACK(EndElemSegment, i);
   }
@@ -2784,6 +2835,7 @@ Result BinaryReader::ReadCodeSection(Offset section_size) {
       ERROR_UNLESS(IsConcreteType(local_type), "expected valid local type");
       CALLBACK(OnLocalDecl, k, num_local_types, local_type);
     }
+    CALLBACK(EndLocalDecls);
 
     if (options_.skip_function_bodies) {
       state_.offset = end_offset;
@@ -2799,13 +2851,13 @@ Result BinaryReader::ReadCodeSection(Offset section_size) {
 
 Result BinaryReader::ReadDataSection(Offset section_size) {
   CALLBACK(BeginDataSection, section_size);
-  Index num_data_segments;
+  CHECK_RESULT(ReadCount(&num_data_segments_, "data segment count"));
-  CHECK_RESULT(ReadCount(&num_data_segments, "data segment count"));
+  CALLBACK(OnDataSegmentCount, num_data_segments_);
-  CALLBACK(OnDataSegmentCount, num_data_segments);
   // If the DataCount section is not present, then data_count_ will be invalid.
-  ERROR_UNLESS(data_count_ == kInvalidIndex || data_count_ == num_data_segments,
+  ERROR_UNLESS(
-               "data segment count does not equal count in DataCount section");
+      data_count_ == kInvalidIndex || data_count_ == num_data_segments_,
-  for (Index i = 0; i < num_data_segments; ++i) {
+      "data segment count does not equal count in DataCount section");
+  for (Index i = 0; i < num_data_segments_; ++i) {
     uint32_t flags;
     CHECK_RESULT(ReadU32Leb128(&flags, "data segment flags"));
     ERROR_IF(flags != 0 && !options_.features.bulk_memory_enabled(),
@@ -2817,7 +2869,6 @@ Result BinaryReader::ReadDataSection(Offset section_size) {
     }
     CALLBACK(BeginDataSegment, i, memory_index, flags);
     if (!(flags & SegPassive)) {
-      ERROR_UNLESS(memories.size() > 0, "no memory to copy data to");
       CALLBACK(BeginDataSegmentInitExpr, i);
       CHECK_RESULT(ReadInitExpr(i));
       CALLBACK(EndDataSegmentInitExpr, i);
@@ -3008,6 +3059,11 @@ Result BinaryReader::ReadModule(const ReadModuleOptions& options) {
   // in case the code section was omitted.
   ERROR_UNLESS(num_function_signatures_ == num_function_bodies_,
                "function signature count != function body count");
+  // This is checked in ReadDataSection, but it must be checked at the end too,
+  // in case the data section was omitted.
+  ERROR_IF(num_data_segments_ == 0 && data_count_ != kInvalidIndex &&
+               data_count_ != 0,
+           "Data section missing but DataCount non-zero");
   CALLBACK0(EndModule);
 
   return Result::Ok;

third_party/wasm2c/src/binary-writer-spec.cc

@@ -256,6 +256,14 @@ void BinaryWriterSpec::WriteConst(const Const& const_) {
       break;
     }
 
+    case Type::ExnRef: {
+      WriteString("exnref");
+      WriteSeparator();
+      WriteKey("value");
+      WriteRefBits(const_.ref_bits());
+      break;
+    }
+
     case Type::V128: {
       WriteString("v128");
       WriteSeparator();

third_party/wasm2c/src/binary-writer.cc

@@ -64,11 +64,18 @@ void WriteType(Stream* stream, Type type, const char* desc) {
   }
 }
 
-void WriteLimits(Stream* stream, const Limits* limits) {
+void WriteLimitsFlags(Stream* stream, uint32_t flags) {
+  WriteU32Leb128(stream, flags, "limits: flags");
+}
+
+uint32_t ComputeLimitsFlags(const Limits* limits) {
   uint32_t flags = limits->has_max ? WABT_BINARY_LIMITS_HAS_MAX_FLAG : 0;
   flags |= limits->is_shared ? WABT_BINARY_LIMITS_IS_SHARED_FLAG : 0;
   flags |= limits->is_64 ? WABT_BINARY_LIMITS_IS_64_FLAG : 0;
-  WriteU32Leb128(stream, flags, "limits: flags");
+  return flags;
+}
+
+void WriteLimitsData(Stream* stream, const Limits* limits) {
   if (limits->is_64) {
     WriteU64Leb128(stream, limits->initial, "limits: initial");
     if (limits->has_max) {
@@ -694,7 +701,7 @@ void BinaryWriter::WriteLoadStoreExpr(const Func* func,
   } else {
     stream_->WriteU8(log2_u32(align), "alignment");
   }
-  WriteU32Leb128(stream_, typed_expr->offset, desc);
+  WriteU64Leb128(stream_, typed_expr->offset, desc);
 }
 
 template <typename T>
@@ -893,13 +900,13 @@ void BinaryWriter::WriteExpr(const Func* func, const Expr* expr) {
       WriteOpcode(stream_, Opcode::End);
       break;
     case ExprType::MemoryCopy: {
-      Index srcmemidx =
-          module_->GetMemoryIndex(cast<MemoryCopyExpr>(expr)->srcmemidx);
       Index destmemidx =
           module_->GetMemoryIndex(cast<MemoryCopyExpr>(expr)->destmemidx);
+      Index srcmemidx =
+          module_->GetMemoryIndex(cast<MemoryCopyExpr>(expr)->srcmemidx);
       WriteOpcode(stream_, Opcode::MemoryCopy);
-      WriteU32Leb128(stream_, srcmemidx, "memory.copy srcmemidx");
       WriteU32Leb128(stream_, destmemidx, "memory.copy destmemidx");
+      WriteU32Leb128(stream_, srcmemidx, "memory.copy srcmemidx");
       break;
     }
     case ExprType::DataDrop: {
@@ -1157,11 +1164,19 @@ void BinaryWriter::WriteFunc(const Func* func) {
 
 void BinaryWriter::WriteTable(const Table* table) {
   WriteType(stream_, table->elem_type);
-  WriteLimits(stream_, &table->elem_limits);
+  WriteLimitsFlags(stream_, ComputeLimitsFlags(&table->elem_limits));
+  WriteLimitsData(stream_, &table->elem_limits);
 }
 
 void BinaryWriter::WriteMemory(const Memory* memory) {
-  WriteLimits(stream_, &memory->page_limits);
+  uint32_t flags = ComputeLimitsFlags(&memory->page_limits);
+  const bool custom_page_size = memory->page_size != WABT_DEFAULT_PAGE_SIZE;
+  flags |= custom_page_size ? WABT_BINARY_LIMITS_HAS_CUSTOM_PAGE_SIZE_FLAG : 0;
+  WriteLimitsFlags(stream_, flags);
+  WriteLimitsData(stream_, &memory->page_limits);
+  if (custom_page_size) {
+    WriteU32Leb128(stream_, log2_u32(memory->page_size), "memory page size");
+  }
 }
 
 void BinaryWriter::WriteGlobalHeader(const Global* global) {
@@ -1197,13 +1212,16 @@ void BinaryWriter::WriteRelocSection(const RelocSection* reloc_section) {
       case RelocType::MemoryAddressRelSLEB64:
       case RelocType::MemoryAddressI32:
       case RelocType::MemoryAddressI64:
+      case RelocType::MemoryAddressLocRelI32:
       case RelocType::FunctionOffsetI32:
+      case RelocType::FunctionOffsetI64:
       case RelocType::SectionOffsetI32:
       case RelocType::MemoryAddressTLSSLEB:
-      case RelocType::MemoryAddressTLSI32:
+      case RelocType::MemoryAddressTLSSLEB64:
         WriteU32Leb128(stream_, reloc.addend, "reloc addend");
         break;
       case RelocType::FuncIndexLEB:
+      case RelocType::FuncIndexI32:
       case RelocType::TableIndexSLEB:
       case RelocType::TableIndexSLEB64:
       case RelocType::TableIndexI32:
@@ -1212,6 +1230,7 @@ void BinaryWriter::WriteRelocSection(const RelocSection* reloc_section) {
       case RelocType::GlobalIndexLEB:
       case RelocType::TagIndexLEB:
       case RelocType::TableIndexRelSLEB:
+      case RelocType::TableIndexRelSLEB64:
       case RelocType::TableNumberLEB:
         break;
       default:
@@ -1670,6 +1689,23 @@ Result BinaryWriter::WriteModule() {
     EndSection();
   }
 
+  for (const Custom& custom : module_->customs) {
+    // These custom sections are already specially handled by BinaryWriter, so
+    // we don't want to double-write.
+    if ((custom.name == WABT_BINARY_SECTION_NAME &&
+         options_.write_debug_names) ||
+        (custom.name.rfind(WABT_BINARY_SECTION_RELOC) == 0 &&
+         options_.relocatable) ||
+        (custom.name == WABT_BINARY_SECTION_LINKING && options_.relocatable) ||
+        (custom.name.find(WABT_BINARY_SECTION_CODE_METADATA) == 0 &&
+         options_.features.code_metadata_enabled())) {
+      continue;
+    }
+    BeginCustomSection(custom.name.data());
+    stream_->WriteData(custom.data, "custom data");
+    EndSection();
+  }
+
   if (options_.write_debug_names) {
     std::vector<std::string> index_to_name;
 

third_party/wasm2c/src/binary.cc

@@ -54,6 +54,7 @@ const char* NameSubsectionName[] = {
     "global",
     "elemseg",
     "dataseg",
+    "field",
     "tag",
 };
 // clang-format on

third_party/wasm2c/src/common.cc

@@ -43,30 +43,49 @@ Reloc::Reloc(RelocType type, Offset offset, Index index, int32_t addend)
 const char* g_kind_name[] = {"func", "table", "memory", "global", "tag"};
 WABT_STATIC_ASSERT(WABT_ARRAY_SIZE(g_kind_name) == kExternalKindCount);
 
+// clang-format off
 const char* g_reloc_type_name[] = {
-    "R_WASM_FUNCTION_INDEX_LEB",   "R_WASM_TABLE_INDEX_SLEB",
+    "R_WASM_FUNCTION_INDEX_LEB",
-    "R_WASM_TABLE_INDEX_I32",      "R_WASM_MEMORY_ADDR_LEB",
+    "R_WASM_TABLE_INDEX_SLEB",
-    "R_WASM_MEMORY_ADDR_SLEB",     "R_WASM_MEMORY_ADDR_I32",
+    "R_WASM_TABLE_INDEX_I32",
-    "R_WASM_TYPE_INDEX_LEB",       "R_WASM_GLOBAL_INDEX_LEB",
+    "R_WASM_MEMORY_ADDR_LEB",
-    "R_WASM_FUNCTION_OFFSET_I32",  "R_WASM_SECTION_OFFSET_I32",
+    "R_WASM_MEMORY_ADDR_SLEB",
-    "R_WASM_TAG_INDEX_LEB",        "R_WASM_MEMORY_ADDR_REL_SLEB",
+    "R_WASM_MEMORY_ADDR_I32",
-    "R_WASM_TABLE_INDEX_REL_SLEB", "R_WASM_GLOBAL_INDEX_I32",
+    "R_WASM_TYPE_INDEX_LEB",
-    "R_WASM_MEMORY_ADDR_LEB64",    "R_WASM_MEMORY_ADDR_SLEB64",
+    "R_WASM_GLOBAL_INDEX_LEB",
-    "R_WASM_MEMORY_ADDR_I64",      "R_WASM_MEMORY_ADDR_REL_SLEB64",
+    "R_WASM_FUNCTION_OFFSET_I32",
-    "R_WASM_TABLE_INDEX_SLEB64",   "R_WASM_TABLE_INDEX_I64",
+    "R_WASM_SECTION_OFFSET_I32",
-    "R_WASM_TABLE_NUMBER_LEB",     "R_WASM_MEMORY_ADDR_TLS_SLEB",
+    "R_WASM_TAG_INDEX_LEB",
-    "R_WASM_MEMORY_ADDR_TLS_I32",
+    "R_WASM_MEMORY_ADDR_REL_SLEB",
+    "R_WASM_TABLE_INDEX_REL_SLEB",
+    "R_WASM_GLOBAL_INDEX_I32",
+    "R_WASM_MEMORY_ADDR_LEB64",
+    "R_WASM_MEMORY_ADDR_SLEB64",
+    "R_WASM_MEMORY_ADDR_I64",
+    "R_WASM_MEMORY_ADDR_REL_SLEB64",
+    "R_WASM_TABLE_INDEX_SLEB64",
+    "R_WASM_TABLE_INDEX_I64",
+    "R_WASM_TABLE_NUMBER_LEB",
+    "R_WASM_MEMORY_ADDR_TLS_SLEB",
+    "R_WASM_FUNCTION_OFFSET_I64",
+    "R_WASM_MEMORY_ADDR_LOCREL_I32",
+    "R_WASM_TABLE_INDEX_REL_SLEB64",
+    "R_WASM_MEMORY_ADDR_TLS_SLEB64",
+    "R_WASM_FUNCTION_INDEX_I32",
 };
 WABT_STATIC_ASSERT(WABT_ARRAY_SIZE(g_reloc_type_name) == kRelocTypeCount);
+// clang-format on
 
-static Result ReadStdin(std::vector<uint8_t>* out_data) {
+static Result ReadAll(FILE* stream,
+                      const char* name,
+                      std::vector<uint8_t>* out_data) {
   out_data->resize(0);
   uint8_t buffer[4096];
   while (true) {
-    size_t bytes_read = fread(buffer, 1, sizeof(buffer), stdin);
+    size_t bytes_read = fread(buffer, 1, sizeof(buffer), stream);
     if (bytes_read == 0) {
-      if (ferror(stdin)) {
+      if (ferror(stream)) {
-        fprintf(stderr, "error reading from stdin: %s\n", strerror(errno));
+        fprintf(stderr, "error reading from %s: %s\n", name, strerror(errno));
         return Result::Error;
       }
       return Result::Ok;
@@ -82,7 +101,7 @@ Result ReadFile(std::string_view filename, std::vector<uint8_t>* out_data) {
   const char* filename_cstr = filename_str.c_str();
 
   if (filename == "-") {
-    return ReadStdin(out_data);
+    return ReadAll(stdin, "stdin", out_data);
   }
 
   struct stat statbuf;
@@ -91,8 +110,8 @@ Result ReadFile(std::string_view filename, std::vector<uint8_t>* out_data) {
     return Result::Error;
   }
 
-  if (!(statbuf.st_mode & S_IFREG)) {
+  if (statbuf.st_mode & S_IFDIR) {
-    fprintf(stderr, "%s: not a regular file\n", filename_cstr);
+    fprintf(stderr, "%s: is a directory\n", filename_cstr);
     return Result::Error;
   }
 
@@ -103,9 +122,11 @@ Result ReadFile(std::string_view filename, std::vector<uint8_t>* out_data) {
   }
 
   if (fseek(infile, 0, SEEK_END) < 0) {
-    perror("fseek to end failed");
+    // not seekable, so we can't pre-allocate out_data, but let's try and read
+    // it anyway (for pipes, sockets, etc.)
+    auto res = ReadAll(infile, filename_cstr, out_data);
     fclose(infile);
-    return Result::Error;
+    return res;
   }
 
   long size = ftell(infile);

third_party/wasm2c/src/config.h.in

@@ -208,24 +208,11 @@ inline int Ctz(unsigned __int64 mask) {
 #endif
 }
 
-#if _M_ARM64
-//https://stackoverflow.com/a/70012905
-template <typename T>
-int BrianKernighanPopcount(T value) {
-  int count;
-  for(count = 0; value; count++)
-  {
-    value &= value - 1;
-  }
-  return count;
-}
-#endif
-
 inline int Popcount(unsigned long value) {
   #if _M_X64 || _M_IX86
   return __popcnt(value);
 #elif _M_ARM64
-  return BrianKernighanPopcount(value);
+  return _CountOneBits(value);
 #else
 #error unexpected architecture
 #endif
@@ -245,7 +232,7 @@ inline int Popcount(unsigned __int64 value) {
 #elif _M_IX86
   return Popcount(HighDword(value)) + Popcount(LowDword(value));
 #elif _M_ARM64
-  return BrianKernighanPopcount(value);
+  return _CountOneBits64(value);
 #else
 #error unexpected architecture
 #endif

third_party/wasm2c/src/decompiler.cc

@@ -527,7 +527,10 @@ struct Decompiler {
           ifs.precedence = Precedence::If;
           return std::move(ifs);
         } else {
-          auto s = cat("if (", ifs.v[0], ") { ", thenp.v[0], " }");
+          auto s = cat("if (", ifs.v[0], ") {");
+          if (!thenp.v.empty())
+            s += cat(" ", thenp.v[0], " ");
+          s += "}";
           if (elsep)
             s += cat(" else { ", elsep->v[0], " }");
           return Value{{std::move(s)}, Precedence::If};
@@ -699,7 +702,7 @@ struct Decompiler {
     size_t line_start = 0;
     static const char s_hexdigits[] = "0123456789abcdef";
     for (auto c : in) {
-      if (c >= ' ' && c <= '~') {
+      if (c >= ' ' && c <= '~' && c != '"' && c != '\\') {
         s += c;
       } else {
         s += '\\';
@@ -770,7 +773,11 @@ struct Decompiler {
 
     // Data.
     for (auto dat : mc.module.data_segments) {
-      s += cat("data ", dat->name, "(offset: ", InitExp(dat->offset), ") =");
+      s += cat("data ", dat->name,
+               dat->kind != SegmentKind::Passive
+                   ? cat("(offset: ", InitExp(dat->offset))
+                   : "(passive",
+               ") =");
       auto ds = BinaryToString(dat->data);
       if (ds.size() > target_exp_width / 2) {
         s += "\n";
@@ -819,9 +826,9 @@ struct Decompiler {
         }
       }
       if (is_import) {
-        s += ";";
+        s += cat("; // func", std::to_string(func_index));
       } else {
-        s += " {\n";
+        s += cat(" { // func", std::to_string(func_index), "\n");
         auto val = DecompileExpr(ast.exp_stack[0], nullptr);
         IndentValue(val, indent_amount, {});
         for (auto& stat : val.v) {

third_party/wasm2c/src/error-formatter.cc

@@ -62,7 +62,10 @@ std::string FormatError(const Error& error,
     result += '\n';
     result += indent_str;
 
-    size_t num_spaces = (loc.first_column - 1) - source_line.column_offset;
+    size_t num_spaces = 0;
+    if (loc.first_column > source_line.column_offset) {
+      num_spaces = (loc.first_column - 1) - source_line.column_offset;
+    }
     size_t num_carets = loc.last_column - loc.first_column;
     num_carets = std::min(num_carets, source_line.line.size() - num_spaces);
     num_carets = std::max<size_t>(num_carets, 1);

third_party/wasm2c/src/leb128.cc

@@ -239,8 +239,8 @@ size_t ReadU64Leb128(const uint8_t* p,
     *out_value = LEB128_9(uint64_t);
     return 9;
   } else if (p + 9 < end && (p[9] & 0x80) == 0) {
-    // The top bits set represent values > 32 bits.
+    // The top bits set represent values > 64 bits.
-    if (p[9] & 0xf0) {
+    if (p[9] & 0xfe) {
       return 0;
     }
     *out_value = LEB128_10(uint64_t);

third_party/wasm2c/src/lexer-keywords.txt

@@ -18,6 +18,7 @@ struct TokenInfo {
 };
 %%
 array, Type::Array, TokenType::Array
+after, TokenType::After
 assert_exception, TokenType::AssertException
 assert_exhaustion, TokenType::AssertExhaustion
 assert_invalid, TokenType::AssertInvalid
@@ -26,6 +27,7 @@ assert_return, TokenType::AssertReturn
 assert_trap, TokenType::AssertTrap
 assert_unlinkable, TokenType::AssertUnlinkable
 atomic.fence, TokenType::AtomicFence, Opcode::AtomicFence
+before, TokenType::Before
 binary, TokenType::Bin
 block, TokenType::Block, Opcode::Block
 br_if, TokenType::BrIf, Opcode::BrIf
@@ -36,6 +38,7 @@ call_ref, TokenType::CallRef, Opcode::CallRef
 call, TokenType::Call, Opcode::Call
 catch, TokenType::Catch, Opcode::Catch
 catch_all, TokenType::CatchAll, Opcode::CatchAll
+code, TokenType::Code
 data.drop, TokenType::DataDrop, Opcode::DataDrop
 data, TokenType::Data
 declare, TokenType::Declare
@@ -50,6 +53,8 @@ end, TokenType::End, Opcode::End
 tag, TokenType::Tag
 extern, Type::ExternRef, TokenType::Extern
 externref, Type::ExternRef
+exn, Type::ExnRef, TokenType::Exn
+exnref, Type::ExnRef
 export, TokenType::Export
 f32.abs, TokenType::Unary, Opcode::F32Abs
 f32.add, TokenType::Binary, Opcode::F32Add
@@ -178,6 +183,7 @@ f64x2, TokenType::F64X2
 field, TokenType::Field
 funcref, Type::FuncRef
 func, Type::FuncRef, TokenType::Func
+function, TokenType::Function
 get, TokenType::Get
 global.get, TokenType::GlobalGet, Opcode::GlobalGet
 global.set, TokenType::GlobalSet, Opcode::GlobalSet
@@ -189,7 +195,7 @@ i16x8.add, TokenType::Binary, Opcode::I16X8Add
 i16x8.all_true, TokenType::Unary, Opcode::I16X8AllTrue
 i16x8.avgr_u, TokenType::Binary, Opcode::I16X8AvgrU
 i16x8.bitmask, TokenType::Unary, Opcode::I16X8Bitmask
-i16x8.dot_i8x16_i7x16_s, TokenType::Binary, Opcode::I16X8DotI8X16I7X16S
+i16x8.relaxed_dot_i8x16_i7x16_s, TokenType::Binary, Opcode::I16X8DotI8X16I7X16S
 i16x8.eq, TokenType::Compare, Opcode::I16X8Eq
 i16x8.extract_lane_s, TokenType::SimdLaneOp, Opcode::I16X8ExtractLaneS
 i16x8.extract_lane_u, TokenType::SimdLaneOp, Opcode::I16X8ExtractLaneU
@@ -315,7 +321,7 @@ i32x4.abs, TokenType::Unary, Opcode::I32X4Abs
 i32x4.add, TokenType::Binary, Opcode::I32X4Add
 i32x4.all_true, TokenType::Unary, Opcode::I32X4AllTrue
 i32x4.bitmask, TokenType::Unary, Opcode::I32X4Bitmask
-i32x4.dot_i8x16_i7x16_add_s, TokenType::Ternary, Opcode::I32X4DotI8X16I7X16AddS
+i32x4.relaxed_dot_i8x16_i7x16_add_s, TokenType::Ternary, Opcode::I32X4DotI8X16I7X16AddS
 i32x4.eq, TokenType::Compare, Opcode::I32X4Eq
 i32x4.extract_lane, TokenType::SimdLaneOp, Opcode::I32X4ExtractLane
 i32x4.ge_s, TokenType::Compare, Opcode::I32X4GeS
@@ -550,6 +556,7 @@ nan:canonical, TokenType::NanCanonical
 nop, TokenType::Nop, Opcode::Nop
 offset, TokenType::Offset
 output, TokenType::Output
+pagesize, TokenType::PageSize
 param, TokenType::Param
 ref, TokenType::Ref
 quote, TokenType::Quote

third_party/wasm2c/src/prebuilt/wasm2c_atomicops_source_declarations.cc

@@ -0,0 +1,444 @@
+const char* s_atomicops_source_declarations = R"w2c_template(#include <stdatomic.h>
+)w2c_template"
+R"w2c_template(
+#ifndef WASM_RT_C11_AVAILABLE
+)w2c_template"
+R"w2c_template(#error "C11 is required for Wasm threads and shared memory support"
+)w2c_template"
+R"w2c_template(#endif
+)w2c_template"
+R"w2c_template(
+#define ATOMIC_ALIGNMENT_CHECK(addr, t1) \
+)w2c_template"
+R"w2c_template(  if (UNLIKELY(addr % sizeof(t1))) {     \
+)w2c_template"
+R"w2c_template(    TRAP(UNALIGNED);                     \
+)w2c_template"
+R"w2c_template(  }
+)w2c_template"
+R"w2c_template(
+#define DEFINE_SHARED_LOAD(name, t1, t2, t3, force_read)          \
+)w2c_template"
+R"w2c_template(  static inline t3 name(wasm_rt_shared_memory_t* mem, u64 addr) { \
+)w2c_template"
+R"w2c_template(    MEMCHECK(mem, addr, t1);                                      \
+)w2c_template"
+R"w2c_template(    t1 result;                                                    \
+)w2c_template"
+R"w2c_template(    result = atomic_load_explicit(                                \
+)w2c_template"
+R"w2c_template(        (_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)),    \
+)w2c_template"
+R"w2c_template(        memory_order_relaxed);                                    \
+)w2c_template"
+R"w2c_template(    force_read(result);                                           \
+)w2c_template"
+R"w2c_template(    return (t3)(t2)result;                                        \
+)w2c_template"
+R"w2c_template(  }
+)w2c_template"
+R"w2c_template(
+DEFINE_SHARED_LOAD(i32_load_shared, u32, u32, u32, FORCE_READ_INT)
+)w2c_template"
+R"w2c_template(DEFINE_SHARED_LOAD(i64_load_shared, u64, u64, u64, FORCE_READ_INT)
+)w2c_template"
+R"w2c_template(DEFINE_SHARED_LOAD(f32_load_shared, f32, f32, f32, FORCE_READ_FLOAT)
+)w2c_template"
+R"w2c_template(DEFINE_SHARED_LOAD(f64_load_shared, f64, f64, f64, FORCE_READ_FLOAT)
+)w2c_template"
+R"w2c_template(DEFINE_SHARED_LOAD(i32_load8_s_shared, s8, s32, u32, FORCE_READ_INT)
+)w2c_template"
+R"w2c_template(DEFINE_SHARED_LOAD(i64_load8_s_shared, s8, s64, u64, FORCE_READ_INT)
+)w2c_template"
+R"w2c_template(DEFINE_SHARED_LOAD(i32_load8_u_shared, u8, u32, u32, FORCE_READ_INT)
+)w2c_template"
+R"w2c_template(DEFINE_SHARED_LOAD(i64_load8_u_shared, u8, u64, u64, FORCE_READ_INT)
+)w2c_template"
+R"w2c_template(DEFINE_SHARED_LOAD(i32_load16_s_shared, s16, s32, u32, FORCE_READ_INT)
+)w2c_template"
+R"w2c_template(DEFINE_SHARED_LOAD(i64_load16_s_shared, s16, s64, u64, FORCE_READ_INT)
+)w2c_template"
+R"w2c_template(DEFINE_SHARED_LOAD(i32_load16_u_shared, u16, u32, u32, FORCE_READ_INT)
+)w2c_template"
+R"w2c_template(DEFINE_SHARED_LOAD(i64_load16_u_shared, u16, u64, u64, FORCE_READ_INT)
+)w2c_template"
+R"w2c_template(DEFINE_SHARED_LOAD(i64_load32_s_shared, s32, s64, u64, FORCE_READ_INT)
+)w2c_template"
+R"w2c_template(DEFINE_SHARED_LOAD(i64_load32_u_shared, u32, u64, u64, FORCE_READ_INT)
+)w2c_template"
+R"w2c_template(
+#define DEFINE_SHARED_STORE(name, t1, t2)                                     \
+)w2c_template"
+R"w2c_template(  static inline void name(wasm_rt_shared_memory_t* mem, u64 addr, t2 value) { \
+)w2c_template"
+R"w2c_template(    MEMCHECK(mem, addr, t1);                                                  \
+)w2c_template"
+R"w2c_template(    t1 wrapped = (t1)value;                                                   \
+)w2c_template"
+R"w2c_template(    atomic_store_explicit(                                                    \
+)w2c_template"
+R"w2c_template(        (_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)), wrapped,       \
+)w2c_template"
+R"w2c_template(        memory_order_relaxed);                                                \
+)w2c_template"
+R"w2c_template(  }
+)w2c_template"
+R"w2c_template(
+DEFINE_SHARED_STORE(i32_store_shared, u32, u32)
+)w2c_template"
+R"w2c_template(DEFINE_SHARED_STORE(i64_store_shared, u64, u64)
+)w2c_template"
+R"w2c_template(DEFINE_SHARED_STORE(f32_store_shared, f32, f32)
+)w2c_template"
+R"w2c_template(DEFINE_SHARED_STORE(f64_store_shared, f64, f64)
+)w2c_template"
+R"w2c_template(DEFINE_SHARED_STORE(i32_store8_shared, u8, u32)
+)w2c_template"
+R"w2c_template(DEFINE_SHARED_STORE(i32_store16_shared, u16, u32)
+)w2c_template"
+R"w2c_template(DEFINE_SHARED_STORE(i64_store8_shared, u8, u64)
+)w2c_template"
+R"w2c_template(DEFINE_SHARED_STORE(i64_store16_shared, u16, u64)
+)w2c_template"
+R"w2c_template(DEFINE_SHARED_STORE(i64_store32_shared, u32, u64)
+)w2c_template"
+R"w2c_template(
+#define DEFINE_ATOMIC_LOAD(name, t1, t2, t3, force_read)                    \
+)w2c_template"
+R"w2c_template(  static inline t3 name(wasm_rt_memory_t* mem, u64 addr) {                  \
+)w2c_template"
+R"w2c_template(    MEMCHECK(mem, addr, t1);                                                \
+)w2c_template"
+R"w2c_template(    ATOMIC_ALIGNMENT_CHECK(addr, t1);                                       \
+)w2c_template"
+R"w2c_template(    t1 result;                                                              \
+)w2c_template"
+R"w2c_template(    wasm_rt_memcpy(&result, MEM_ADDR(mem, addr, sizeof(t1)), sizeof(t1));   \
+)w2c_template"
+R"w2c_template(    force_read(result);                                                     \
+)w2c_template"
+R"w2c_template(    return (t3)(t2)result;                                                  \
+)w2c_template"
+R"w2c_template(  }                                                                         \
+)w2c_template"
+R"w2c_template(  static inline t3 name##_shared(wasm_rt_shared_memory_t* mem, u64 addr) {  \
+)w2c_template"
+R"w2c_template(    MEMCHECK(mem, addr, t1);                                                \
+)w2c_template"
+R"w2c_template(    ATOMIC_ALIGNMENT_CHECK(addr, t1);                                       \
+)w2c_template"
+R"w2c_template(    t1 result;                                                              \
+)w2c_template"
+R"w2c_template(    result =                                                                \
+)w2c_template"
+R"w2c_template(        atomic_load((_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1))); \
+)w2c_template"
+R"w2c_template(    force_read(result);                                                     \
+)w2c_template"
+R"w2c_template(    return (t3)(t2)result;                                                  \
+)w2c_template"
+R"w2c_template(  }
+)w2c_template"
+R"w2c_template(
+DEFINE_ATOMIC_LOAD(i32_atomic_load, u32, u32, u32, FORCE_READ_INT)
+)w2c_template"
+R"w2c_template(DEFINE_ATOMIC_LOAD(i64_atomic_load, u64, u64, u64, FORCE_READ_INT)
+)w2c_template"
+R"w2c_template(DEFINE_ATOMIC_LOAD(i32_atomic_load8_u, u8, u32, u32, FORCE_READ_INT)
+)w2c_template"
+R"w2c_template(DEFINE_ATOMIC_LOAD(i64_atomic_load8_u, u8, u64, u64, FORCE_READ_INT)
+)w2c_template"
+R"w2c_template(DEFINE_ATOMIC_LOAD(i32_atomic_load16_u, u16, u32, u32, FORCE_READ_INT)
+)w2c_template"
+R"w2c_template(DEFINE_ATOMIC_LOAD(i64_atomic_load16_u, u16, u64, u64, FORCE_READ_INT)
+)w2c_template"
+R"w2c_template(DEFINE_ATOMIC_LOAD(i64_atomic_load32_u, u32, u64, u64, FORCE_READ_INT)
+)w2c_template"
+R"w2c_template(
+#define DEFINE_ATOMIC_STORE(name, t1, t2)                                  \
+)w2c_template"
+R"w2c_template(  static inline void name(wasm_rt_memory_t* mem, u64 addr, t2 value) {     \
+)w2c_template"
+R"w2c_template(    MEMCHECK(mem, addr, t1);                                               \
+)w2c_template"
+R"w2c_template(    ATOMIC_ALIGNMENT_CHECK(addr, t1);                                      \
+)w2c_template"
+R"w2c_template(    t1 wrapped = (t1)value;                                                \
+)w2c_template"
+R"w2c_template(    wasm_rt_memcpy(MEM_ADDR(mem, addr, sizeof(t1)), &wrapped, sizeof(t1)); \
+)w2c_template"
+R"w2c_template(  }                                                                        \
+)w2c_template"
+R"w2c_template(  static inline void name##_shared(wasm_rt_shared_memory_t* mem, u64 addr, \
+)w2c_template"
+R"w2c_template(                                   t2 value) {                             \
+)w2c_template"
+R"w2c_template(    MEMCHECK(mem, addr, t1);                                               \
+)w2c_template"
+R"w2c_template(    ATOMIC_ALIGNMENT_CHECK(addr, t1);                                      \
+)w2c_template"
+R"w2c_template(    t1 wrapped = (t1)value;                                                \
+)w2c_template"
+R"w2c_template(    atomic_store((_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)),    \
+)w2c_template"
+R"w2c_template(                 wrapped);                                                 \
+)w2c_template"
+R"w2c_template(  }
+)w2c_template"
+R"w2c_template(
+DEFINE_ATOMIC_STORE(i32_atomic_store, u32, u32)
+)w2c_template"
+R"w2c_template(DEFINE_ATOMIC_STORE(i64_atomic_store, u64, u64)
+)w2c_template"
+R"w2c_template(DEFINE_ATOMIC_STORE(i32_atomic_store8, u8, u32)
+)w2c_template"
+R"w2c_template(DEFINE_ATOMIC_STORE(i32_atomic_store16, u16, u32)
+)w2c_template"
+R"w2c_template(DEFINE_ATOMIC_STORE(i64_atomic_store8, u8, u64)
+)w2c_template"
+R"w2c_template(DEFINE_ATOMIC_STORE(i64_atomic_store16, u16, u64)
+)w2c_template"
+R"w2c_template(DEFINE_ATOMIC_STORE(i64_atomic_store32, u32, u64)
+)w2c_template"
+R"w2c_template(
+#define DEFINE_ATOMIC_RMW(name, opname, op, t1, t2)                      \
+)w2c_template"
+R"w2c_template(  static inline t2 name(wasm_rt_memory_t* mem, u64 addr, t2 value) {     \
+)w2c_template"
+R"w2c_template(    MEMCHECK(mem, addr, t1);                                             \
+)w2c_template"
+R"w2c_template(    ATOMIC_ALIGNMENT_CHECK(addr, t1);                                    \
+)w2c_template"
+R"w2c_template(    t1 wrapped = (t1)value;                                              \
+)w2c_template"
+R"w2c_template(    t1 ret;                                                              \
+)w2c_template"
+R"w2c_template(    wasm_rt_memcpy(&ret, MEM_ADDR(mem, addr, sizeof(t1)), sizeof(t1));   \
+)w2c_template"
+R"w2c_template(    ret = ret op wrapped;                                                \
+)w2c_template"
+R"w2c_template(    wasm_rt_memcpy(MEM_ADDR(mem, addr, sizeof(t1)), &ret, sizeof(t1));   \
+)w2c_template"
+R"w2c_template(    return (t2)ret;                                                      \
+)w2c_template"
+R"w2c_template(  }                                                                      \
+)w2c_template"
+R"w2c_template(  static inline t2 name##_shared(wasm_rt_shared_memory_t* mem, u64 addr, \
+)w2c_template"
+R"w2c_template(                                 t2 value) {                             \
+)w2c_template"
+R"w2c_template(    MEMCHECK(mem, addr, t1);                                             \
+)w2c_template"
+R"w2c_template(    ATOMIC_ALIGNMENT_CHECK(addr, t1);                                    \
+)w2c_template"
+R"w2c_template(    t1 wrapped = (t1)value;                                              \
+)w2c_template"
+R"w2c_template(    t1 ret = atomic_##opname(                                            \
+)w2c_template"
+R"w2c_template(        (_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)), wrapped); \
+)w2c_template"
+R"w2c_template(    return (t2)ret;                                                      \
+)w2c_template"
+R"w2c_template(  }
+)w2c_template"
+R"w2c_template(
+DEFINE_ATOMIC_RMW(i32_atomic_rmw8_add_u, fetch_add, +, u8, u32)
+)w2c_template"
+R"w2c_template(DEFINE_ATOMIC_RMW(i32_atomic_rmw16_add_u, fetch_add, +, u16, u32)
+)w2c_template"
+R"w2c_template(DEFINE_ATOMIC_RMW(i32_atomic_rmw_add, fetch_add, +, u32, u32)
+)w2c_template"
+R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw8_add_u, fetch_add, +, u8, u64)
+)w2c_template"
+R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw16_add_u, fetch_add, +, u16, u64)
+)w2c_template"
+R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw32_add_u, fetch_add, +, u32, u64)
+)w2c_template"
+R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw_add, fetch_add, +, u64, u64)
+)w2c_template"
+R"w2c_template(
+DEFINE_ATOMIC_RMW(i32_atomic_rmw8_sub_u, fetch_sub, -, u8, u32)
+)w2c_template"
+R"w2c_template(DEFINE_ATOMIC_RMW(i32_atomic_rmw16_sub_u, fetch_sub, -, u16, u32)
+)w2c_template"
+R"w2c_template(DEFINE_ATOMIC_RMW(i32_atomic_rmw_sub, fetch_sub, -, u32, u32)
+)w2c_template"
+R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw8_sub_u, fetch_sub, -, u8, u64)
+)w2c_template"
+R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw16_sub_u, fetch_sub, -, u16, u64)
+)w2c_template"
+R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw32_sub_u, fetch_sub, -, u32, u64)
+)w2c_template"
+R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw_sub, fetch_sub, -, u64, u64)
+)w2c_template"
+R"w2c_template(
+DEFINE_ATOMIC_RMW(i32_atomic_rmw8_and_u, fetch_and, &, u8, u32)
+)w2c_template"
+R"w2c_template(DEFINE_ATOMIC_RMW(i32_atomic_rmw16_and_u, fetch_and, &, u16, u32)
+)w2c_template"
+R"w2c_template(DEFINE_ATOMIC_RMW(i32_atomic_rmw_and, fetch_and, &, u32, u32)
+)w2c_template"
+R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw8_and_u, fetch_and, &, u8, u64)
+)w2c_template"
+R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw16_and_u, fetch_and, &, u16, u64)
+)w2c_template"
+R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw32_and_u, fetch_and, &, u32, u64)
+)w2c_template"
+R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw_and, fetch_and, &, u64, u64)
+)w2c_template"
+R"w2c_template(
+DEFINE_ATOMIC_RMW(i32_atomic_rmw8_or_u, fetch_or, |, u8, u32)
+)w2c_template"
+R"w2c_template(DEFINE_ATOMIC_RMW(i32_atomic_rmw16_or_u, fetch_or, |, u16, u32)
+)w2c_template"
+R"w2c_template(DEFINE_ATOMIC_RMW(i32_atomic_rmw_or, fetch_or, |, u32, u32)
+)w2c_template"
+R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw8_or_u, fetch_or, |, u8, u64)
+)w2c_template"
+R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw16_or_u, fetch_or, |, u16, u64)
+)w2c_template"
+R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw32_or_u, fetch_or, |, u32, u64)
+)w2c_template"
+R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw_or, fetch_or, |, u64, u64)
+)w2c_template"
+R"w2c_template(
+DEFINE_ATOMIC_RMW(i32_atomic_rmw8_xor_u, fetch_xor, ^, u8, u32)
+)w2c_template"
+R"w2c_template(DEFINE_ATOMIC_RMW(i32_atomic_rmw16_xor_u, fetch_xor, ^, u16, u32)
+)w2c_template"
+R"w2c_template(DEFINE_ATOMIC_RMW(i32_atomic_rmw_xor, fetch_xor, ^, u32, u32)
+)w2c_template"
+R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw8_xor_u, fetch_xor, ^, u8, u64)
+)w2c_template"
+R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw16_xor_u, fetch_xor, ^, u16, u64)
+)w2c_template"
+R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw32_xor_u, fetch_xor, ^, u32, u64)
+)w2c_template"
+R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw_xor, fetch_xor, ^, u64, u64)
+)w2c_template"
+R"w2c_template(
+#define DEFINE_ATOMIC_XCHG(name, opname, t1, t2)                           \
+)w2c_template"
+R"w2c_template(  static inline t2 name(wasm_rt_memory_t* mem, u64 addr, t2 value) {       \
+)w2c_template"
+R"w2c_template(    MEMCHECK(mem, addr, t1);                                               \
+)w2c_template"
+R"w2c_template(    ATOMIC_ALIGNMENT_CHECK(addr, t1);                                      \
+)w2c_template"
+R"w2c_template(    t1 wrapped = (t1)value;                                                \
+)w2c_template"
+R"w2c_template(    t1 ret;                                                                \
+)w2c_template"
+R"w2c_template(    wasm_rt_memcpy(&ret, MEM_ADDR(mem, addr, sizeof(t1)), sizeof(t1));     \
+)w2c_template"
+R"w2c_template(    wasm_rt_memcpy(MEM_ADDR(mem, addr, sizeof(t1)), &wrapped, sizeof(t1)); \
+)w2c_template"
+R"w2c_template(    return (t2)ret;                                                        \
+)w2c_template"
+R"w2c_template(  }                                                                        \
+)w2c_template"
+R"w2c_template(  static inline t2 name##_shared(wasm_rt_shared_memory_t* mem, u64 addr,   \
+)w2c_template"
+R"w2c_template(                                 t2 value) {                               \
+)w2c_template"
+R"w2c_template(    MEMCHECK(mem, addr, t1);                                               \
+)w2c_template"
+R"w2c_template(    ATOMIC_ALIGNMENT_CHECK(addr, t1);                                      \
+)w2c_template"
+R"w2c_template(    t1 wrapped = (t1)value;                                                \
+)w2c_template"
+R"w2c_template(    t1 ret = atomic_##opname(                                              \
+)w2c_template"
+R"w2c_template(        (_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)), wrapped);   \
+)w2c_template"
+R"w2c_template(    return (t2)ret;                                                        \
+)w2c_template"
+R"w2c_template(  }
+)w2c_template"
+R"w2c_template(
+DEFINE_ATOMIC_XCHG(i32_atomic_rmw8_xchg_u, exchange, u8, u32)
+)w2c_template"
+R"w2c_template(DEFINE_ATOMIC_XCHG(i32_atomic_rmw16_xchg_u, exchange, u16, u32)
+)w2c_template"
+R"w2c_template(DEFINE_ATOMIC_XCHG(i32_atomic_rmw_xchg, exchange, u32, u32)
+)w2c_template"
+R"w2c_template(DEFINE_ATOMIC_XCHG(i64_atomic_rmw8_xchg_u, exchange, u8, u64)
+)w2c_template"
+R"w2c_template(DEFINE_ATOMIC_XCHG(i64_atomic_rmw16_xchg_u, exchange, u16, u64)
+)w2c_template"
+R"w2c_template(DEFINE_ATOMIC_XCHG(i64_atomic_rmw32_xchg_u, exchange, u32, u64)
+)w2c_template"
+R"w2c_template(DEFINE_ATOMIC_XCHG(i64_atomic_rmw_xchg, exchange, u64, u64)
+)w2c_template"
+R"w2c_template(
+#define DEFINE_ATOMIC_CMP_XCHG(name, t1, t2)                                \
+)w2c_template"
+R"w2c_template(  static inline t1 name(wasm_rt_memory_t* mem, u64 addr, t1 expected,       \
+)w2c_template"
+R"w2c_template(                        t1 replacement) {                                   \
+)w2c_template"
+R"w2c_template(    MEMCHECK(mem, addr, t2);                                                \
+)w2c_template"
+R"w2c_template(    ATOMIC_ALIGNMENT_CHECK(addr, t2);                                       \
+)w2c_template"
+R"w2c_template(    t2 expected_wrapped = (t2)expected;                                     \
+)w2c_template"
+R"w2c_template(    t2 replacement_wrapped = (t2)replacement;                               \
+)w2c_template"
+R"w2c_template(    t2 ret;                                                                 \
+)w2c_template"
+R"w2c_template(    wasm_rt_memcpy(&ret, MEM_ADDR(mem, addr, sizeof(t2)), sizeof(t2));      \
+)w2c_template"
+R"w2c_template(    if (ret == expected_wrapped) {                                          \
+)w2c_template"
+R"w2c_template(      wasm_rt_memcpy(MEM_ADDR(mem, addr, sizeof(t2)), &replacement_wrapped, \
+)w2c_template"
+R"w2c_template(                     sizeof(t2));                                           \
+)w2c_template"
+R"w2c_template(    }                                                                       \
+)w2c_template"
+R"w2c_template(    return (t1)expected_wrapped;                                            \
+)w2c_template"
+R"w2c_template(  }                                                                         \
+)w2c_template"
+R"w2c_template(  static inline t1 name##_shared(wasm_rt_shared_memory_t* mem, u64 addr,    \
+)w2c_template"
+R"w2c_template(                                 t1 expected, t1 replacement) {             \
+)w2c_template"
+R"w2c_template(    MEMCHECK(mem, addr, t2);                                                \
+)w2c_template"
+R"w2c_template(    ATOMIC_ALIGNMENT_CHECK(addr, t2);                                       \
+)w2c_template"
+R"w2c_template(    t2 expected_wrapped = (t2)expected;                                     \
+)w2c_template"
+R"w2c_template(    t2 replacement_wrapped = (t2)replacement;                               \
+)w2c_template"
+R"w2c_template(    atomic_compare_exchange_strong(                                         \
+)w2c_template"
+R"w2c_template(        (_Atomic volatile t2*)MEM_ADDR(mem, addr, sizeof(t2)),              \
+)w2c_template"
+R"w2c_template(        &expected_wrapped, replacement_wrapped);                            \
+)w2c_template"
+R"w2c_template(    return (t1)expected_wrapped;                                            \
+)w2c_template"
+R"w2c_template(  }
+)w2c_template"
+R"w2c_template(
+DEFINE_ATOMIC_CMP_XCHG(i32_atomic_rmw8_cmpxchg_u, u32, u8);
+)w2c_template"
+R"w2c_template(DEFINE_ATOMIC_CMP_XCHG(i32_atomic_rmw16_cmpxchg_u, u32, u16);
+)w2c_template"
+R"w2c_template(DEFINE_ATOMIC_CMP_XCHG(i32_atomic_rmw_cmpxchg, u32, u32);
+)w2c_template"
+R"w2c_template(DEFINE_ATOMIC_CMP_XCHG(i64_atomic_rmw8_cmpxchg_u, u64, u8);
+)w2c_template"
+R"w2c_template(DEFINE_ATOMIC_CMP_XCHG(i64_atomic_rmw16_cmpxchg_u, u64, u16);
+)w2c_template"
+R"w2c_template(DEFINE_ATOMIC_CMP_XCHG(i64_atomic_rmw32_cmpxchg_u, u64, u32);
+)w2c_template"
+R"w2c_template(DEFINE_ATOMIC_CMP_XCHG(i64_atomic_rmw_cmpxchg, u64, u64);
+)w2c_template"
+R"w2c_template(
+#define atomic_fence() atomic_thread_fence(memory_order_seq_cst)
+)w2c_template"
+;

third_party/wasm2c/src/prebuilt/wasm2c_simd_source_declarations.cc

@@ -1,6 +1,10 @@
-const char* s_simd_source_declarations = R"w2c_template(#ifdef __x86_64__
+const char* s_simd_source_declarations = R"w2c_template(#if defined(__GNUC__) && defined(__x86_64__)
 )w2c_template"
-R"w2c_template(#define SIMD_FORCE_READ(var) wasm_asm("" ::"x"(var));
+R"w2c_template(#define SIMD_FORCE_READ(var) __asm__("" ::"x"(var));
+)w2c_template"
+R"w2c_template(#elif defined(__GNUC__) && defined(__aarch64__)
+)w2c_template"
+R"w2c_template(#define SIMD_FORCE_READ(var) __asm__("" ::"w"(var));
 )w2c_template"
 R"w2c_template(#else
 )w2c_template"
@@ -17,7 +21,7 @@ R"w2c_template(  static inline v128 name(wasm_rt_memory_t* mem, u64 addr) { \
 )w2c_template"
 R"w2c_template(    MEMCHECK(mem, addr, t);                                  \
 )w2c_template"
-R"w2c_template(    v128 result = func((v128*)&mem->data[addr]);             \
+R"w2c_template(    v128 result = func(MEM_ADDR(mem, addr, sizeof(t)));      \
 )w2c_template"
 R"w2c_template(    SIMD_FORCE_READ(result);                                 \
 )w2c_template"
@@ -32,7 +36,7 @@ R"w2c_template(  static inline v128 name(wasm_rt_memory_t* mem, u64 addr, v128 v
 )w2c_template"
 R"w2c_template(    MEMCHECK(mem, addr, t);                                            \
 )w2c_template"
-R"w2c_template(    v128 result = func((v128*)&mem->data[addr], vec, lane);            \
+R"w2c_template(    v128 result = func(MEM_ADDR(mem, addr, sizeof(t)), vec, lane);     \
 )w2c_template"
 R"w2c_template(    SIMD_FORCE_READ(result);                                           \
 )w2c_template"
@@ -47,7 +51,7 @@ R"w2c_template(  static inline void name(wasm_rt_memory_t* mem, u64 addr, v128 v
 )w2c_template"
 R"w2c_template(    MEMCHECK(mem, addr, t);                                              \
 )w2c_template"
-R"w2c_template(    simde_wasm_v128_store((v128*)&mem->data[addr], value);               \
+R"w2c_template(    simde_wasm_v128_store(MEM_ADDR(mem, addr, sizeof(t)), value);        \
 )w2c_template"
 R"w2c_template(  }
 )w2c_template"
@@ -58,14 +62,45 @@ R"w2c_template(  static inline void name(wasm_rt_memory_t* mem, u64 addr, v128 v
 )w2c_template"
 R"w2c_template(    MEMCHECK(mem, addr, t);                                              \
 )w2c_template"
-R"w2c_template(    func((v128*)&mem->data[addr], value, lane);                          \
+R"w2c_template(    func(MEM_ADDR(mem, addr, sizeof(t)), value, lane);                   \
 )w2c_template"
 R"w2c_template(  }
 )w2c_template"
 R"w2c_template(
 // clang-format off
 )w2c_template"
-R"w2c_template(DEFINE_SIMD_LOAD_FUNC(v128_load, simde_wasm_v128_load, v128)
+R"w2c_template(#if WABT_BIG_ENDIAN
+)w2c_template"
+R"w2c_template(static inline v128 v128_impl_load32_zero(const void* a) {
+)w2c_template"
+R"w2c_template(  return simde_wasm_i8x16_swizzle(
+)w2c_template"
+R"w2c_template(      simde_wasm_v128_load32_zero(a),
+)w2c_template"
+R"w2c_template(      simde_wasm_i8x16_const(12,13,14,15,8,9,10,11,4,5,6,7,0,1,2,3));
+)w2c_template"
+R"w2c_template(}
+)w2c_template"
+R"w2c_template(static inline v128 v128_impl_load64_zero(const void* a) {
+)w2c_template"
+R"w2c_template(  return simde_wasm_i8x16_swizzle(
+)w2c_template"
+R"w2c_template(      simde_wasm_v128_load64_zero(a),
+)w2c_template"
+R"w2c_template(      simde_wasm_i8x16_const(8,9,10,11,12,13,14,15,0,1,2,3,4,5,6,7));
+)w2c_template"
+R"w2c_template(}
+)w2c_template"
+R"w2c_template(#else
+)w2c_template"
+R"w2c_template(#define v128_impl_load32_zero simde_wasm_v128_load32_zero
+)w2c_template"
+R"w2c_template(#define v128_impl_load64_zero simde_wasm_v128_load64_zero
+)w2c_template"
+R"w2c_template(#endif
+)w2c_template"
+R"w2c_template(
+DEFINE_SIMD_LOAD_FUNC(v128_load, simde_wasm_v128_load, v128)
 )w2c_template"
 R"w2c_template(
 DEFINE_SIMD_LOAD_FUNC(v128_load8_splat, simde_wasm_v128_load8_splat, u8)
@@ -90,12 +125,76 @@ R"w2c_template(DEFINE_SIMD_LOAD_FUNC(i64x2_load32x2, simde_wasm_i64x2_load32x2,
 R"w2c_template(DEFINE_SIMD_LOAD_FUNC(u64x2_load32x2, simde_wasm_u64x2_load32x2, u64)
 )w2c_template"
 R"w2c_template(
-DEFINE_SIMD_LOAD_FUNC(v128_load32_zero, simde_wasm_v128_load32_zero, u32)
+DEFINE_SIMD_LOAD_FUNC(v128_load32_zero, v128_impl_load32_zero, u32)
 )w2c_template"
-R"w2c_template(DEFINE_SIMD_LOAD_FUNC(v128_load64_zero, simde_wasm_v128_load64_zero, u64)
+R"w2c_template(DEFINE_SIMD_LOAD_FUNC(v128_load64_zero, v128_impl_load64_zero, u64)
 )w2c_template"
 R"w2c_template(
-DEFINE_SIMD_LOAD_LANE(v128_load8_lane0, simde_wasm_v128_load8_lane, u8, 0)
+#if WABT_BIG_ENDIAN
+)w2c_template"
+R"w2c_template(DEFINE_SIMD_LOAD_LANE(v128_load8_lane0, simde_wasm_v128_load8_lane, u8, 15)
+)w2c_template"
+R"w2c_template(DEFINE_SIMD_LOAD_LANE(v128_load8_lane1, simde_wasm_v128_load8_lane, u8, 14)
+)w2c_template"
+R"w2c_template(DEFINE_SIMD_LOAD_LANE(v128_load8_lane2, simde_wasm_v128_load8_lane, u8, 13)
+)w2c_template"
+R"w2c_template(DEFINE_SIMD_LOAD_LANE(v128_load8_lane3, simde_wasm_v128_load8_lane, u8, 12)
+)w2c_template"
+R"w2c_template(DEFINE_SIMD_LOAD_LANE(v128_load8_lane4, simde_wasm_v128_load8_lane, u8, 11)
+)w2c_template"
+R"w2c_template(DEFINE_SIMD_LOAD_LANE(v128_load8_lane5, simde_wasm_v128_load8_lane, u8, 10)
+)w2c_template"
+R"w2c_template(DEFINE_SIMD_LOAD_LANE(v128_load8_lane6, simde_wasm_v128_load8_lane, u8, 9)
+)w2c_template"
+R"w2c_template(DEFINE_SIMD_LOAD_LANE(v128_load8_lane7, simde_wasm_v128_load8_lane, u8, 8)
+)w2c_template"
+R"w2c_template(DEFINE_SIMD_LOAD_LANE(v128_load8_lane8, simde_wasm_v128_load8_lane, u8, 7)
+)w2c_template"
+R"w2c_template(DEFINE_SIMD_LOAD_LANE(v128_load8_lane9, simde_wasm_v128_load8_lane, u8, 6)
+)w2c_template"
+R"w2c_template(DEFINE_SIMD_LOAD_LANE(v128_load8_lane10, simde_wasm_v128_load8_lane, u8, 5)
+)w2c_template"
+R"w2c_template(DEFINE_SIMD_LOAD_LANE(v128_load8_lane11, simde_wasm_v128_load8_lane, u8, 4)
+)w2c_template"
+R"w2c_template(DEFINE_SIMD_LOAD_LANE(v128_load8_lane12, simde_wasm_v128_load8_lane, u8, 3)
+)w2c_template"
+R"w2c_template(DEFINE_SIMD_LOAD_LANE(v128_load8_lane13, simde_wasm_v128_load8_lane, u8, 2)
+)w2c_template"
+R"w2c_template(DEFINE_SIMD_LOAD_LANE(v128_load8_lane14, simde_wasm_v128_load8_lane, u8, 1)
+)w2c_template"
+R"w2c_template(DEFINE_SIMD_LOAD_LANE(v128_load8_lane15, simde_wasm_v128_load8_lane, u8, 0)
+)w2c_template"
+R"w2c_template(DEFINE_SIMD_LOAD_LANE(v128_load16_lane0, simde_wasm_v128_load16_lane, u16, 7)
+)w2c_template"
+R"w2c_template(DEFINE_SIMD_LOAD_LANE(v128_load16_lane1, simde_wasm_v128_load16_lane, u16, 6)
+)w2c_template"
+R"w2c_template(DEFINE_SIMD_LOAD_LANE(v128_load16_lane2, simde_wasm_v128_load16_lane, u16, 5)
+)w2c_template"
+R"w2c_template(DEFINE_SIMD_LOAD_LANE(v128_load16_lane3, simde_wasm_v128_load16_lane, u16, 4)
+)w2c_template"
+R"w2c_template(DEFINE_SIMD_LOAD_LANE(v128_load16_lane4, simde_wasm_v128_load16_lane, u16, 3)
+)w2c_template"
+R"w2c_template(DEFINE_SIMD_LOAD_LANE(v128_load16_lane5, simde_wasm_v128_load16_lane, u16, 2)
+)w2c_template"
+R"w2c_template(DEFINE_SIMD_LOAD_LANE(v128_load16_lane6, simde_wasm_v128_load16_lane, u16, 1)
+)w2c_template"
+R"w2c_template(DEFINE_SIMD_LOAD_LANE(v128_load16_lane7, simde_wasm_v128_load16_lane, u16, 0)
+)w2c_template"
+R"w2c_template(DEFINE_SIMD_LOAD_LANE(v128_load32_lane0, simde_wasm_v128_load32_lane, u32, 3)
+)w2c_template"
+R"w2c_template(DEFINE_SIMD_LOAD_LANE(v128_load32_lane1, simde_wasm_v128_load32_lane, u32, 2)
+)w2c_template"
+R"w2c_template(DEFINE_SIMD_LOAD_LANE(v128_load32_lane2, simde_wasm_v128_load32_lane, u32, 1)
+)w2c_template"
+R"w2c_template(DEFINE_SIMD_LOAD_LANE(v128_load32_lane3, simde_wasm_v128_load32_lane, u32, 0)
+)w2c_template"
+R"w2c_template(DEFINE_SIMD_LOAD_LANE(v128_load64_lane0, simde_wasm_v128_load64_lane, u64, 1)
+)w2c_template"
+R"w2c_template(DEFINE_SIMD_LOAD_LANE(v128_load64_lane1, simde_wasm_v128_load64_lane, u64, 0)
+)w2c_template"
+R"w2c_template(#else
+)w2c_template"
+R"w2c_template(DEFINE_SIMD_LOAD_LANE(v128_load8_lane0, simde_wasm_v128_load8_lane, u8, 0)
 )w2c_template"
 R"w2c_template(DEFINE_SIMD_LOAD_LANE(v128_load8_lane1, simde_wasm_v128_load8_lane, u8, 1)
 )w2c_template"
@@ -155,11 +254,77 @@ R"w2c_template(DEFINE_SIMD_LOAD_LANE(v128_load64_lane0, simde_wasm_v128_load64_l
 )w2c_template"
 R"w2c_template(DEFINE_SIMD_LOAD_LANE(v128_load64_lane1, simde_wasm_v128_load64_lane, u64, 1)
 )w2c_template"
+R"w2c_template(#endif
+)w2c_template"
 R"w2c_template(
 DEFINE_SIMD_STORE(v128_store, v128)
 )w2c_template"
 R"w2c_template(
-DEFINE_SIMD_STORE_LANE(v128_store8_lane0, simde_wasm_v128_store8_lane, u8, 0)
+#if WABT_BIG_ENDIAN
+)w2c_template"
+R"w2c_template(DEFINE_SIMD_STORE_LANE(v128_store8_lane0, simde_wasm_v128_store8_lane, u8, 15)
+)w2c_template"
+R"w2c_template(DEFINE_SIMD_STORE_LANE(v128_store8_lane1, simde_wasm_v128_store8_lane, u8, 14)
+)w2c_template"
+R"w2c_template(DEFINE_SIMD_STORE_LANE(v128_store8_lane2, simde_wasm_v128_store8_lane, u8, 13)
+)w2c_template"
+R"w2c_template(DEFINE_SIMD_STORE_LANE(v128_store8_lane3, simde_wasm_v128_store8_lane, u8, 12)
+)w2c_template"
+R"w2c_template(DEFINE_SIMD_STORE_LANE(v128_store8_lane4, simde_wasm_v128_store8_lane, u8, 11)
+)w2c_template"
+R"w2c_template(DEFINE_SIMD_STORE_LANE(v128_store8_lane5, simde_wasm_v128_store8_lane, u8, 10)
+)w2c_template"
+R"w2c_template(DEFINE_SIMD_STORE_LANE(v128_store8_lane6, simde_wasm_v128_store8_lane, u8, 9)
+)w2c_template"
+R"w2c_template(DEFINE_SIMD_STORE_LANE(v128_store8_lane7, simde_wasm_v128_store8_lane, u8, 8)
+)w2c_template"
+R"w2c_template(DEFINE_SIMD_STORE_LANE(v128_store8_lane8, simde_wasm_v128_store8_lane, u8, 7)
+)w2c_template"
+R"w2c_template(DEFINE_SIMD_STORE_LANE(v128_store8_lane9, simde_wasm_v128_store8_lane, u8, 6)
+)w2c_template"
+R"w2c_template(DEFINE_SIMD_STORE_LANE(v128_store8_lane10, simde_wasm_v128_store8_lane, u8, 5)
+)w2c_template"
+R"w2c_template(DEFINE_SIMD_STORE_LANE(v128_store8_lane11, simde_wasm_v128_store8_lane, u8, 4)
+)w2c_template"
+R"w2c_template(DEFINE_SIMD_STORE_LANE(v128_store8_lane12, simde_wasm_v128_store8_lane, u8, 3)
+)w2c_template"
+R"w2c_template(DEFINE_SIMD_STORE_LANE(v128_store8_lane13, simde_wasm_v128_store8_lane, u8, 2)
+)w2c_template"
+R"w2c_template(DEFINE_SIMD_STORE_LANE(v128_store8_lane14, simde_wasm_v128_store8_lane, u8, 1)
+)w2c_template"
+R"w2c_template(DEFINE_SIMD_STORE_LANE(v128_store8_lane15, simde_wasm_v128_store8_lane, u8, 0)
+)w2c_template"
+R"w2c_template(DEFINE_SIMD_STORE_LANE(v128_store16_lane0, simde_wasm_v128_store16_lane, u16, 7)
+)w2c_template"
+R"w2c_template(DEFINE_SIMD_STORE_LANE(v128_store16_lane1, simde_wasm_v128_store16_lane, u16, 6)
+)w2c_template"
+R"w2c_template(DEFINE_SIMD_STORE_LANE(v128_store16_lane2, simde_wasm_v128_store16_lane, u16, 5)
+)w2c_template"
+R"w2c_template(DEFINE_SIMD_STORE_LANE(v128_store16_lane3, simde_wasm_v128_store16_lane, u16, 4)
+)w2c_template"
+R"w2c_template(DEFINE_SIMD_STORE_LANE(v128_store16_lane4, simde_wasm_v128_store16_lane, u16, 3)
+)w2c_template"
+R"w2c_template(DEFINE_SIMD_STORE_LANE(v128_store16_lane5, simde_wasm_v128_store16_lane, u16, 2)
+)w2c_template"
+R"w2c_template(DEFINE_SIMD_STORE_LANE(v128_store16_lane6, simde_wasm_v128_store16_lane, u16, 1)
+)w2c_template"
+R"w2c_template(DEFINE_SIMD_STORE_LANE(v128_store16_lane7, simde_wasm_v128_store16_lane, u16, 0)
+)w2c_template"
+R"w2c_template(DEFINE_SIMD_STORE_LANE(v128_store32_lane0, simde_wasm_v128_store32_lane, u32, 3)
+)w2c_template"
+R"w2c_template(DEFINE_SIMD_STORE_LANE(v128_store32_lane1, simde_wasm_v128_store32_lane, u32, 2)
+)w2c_template"
+R"w2c_template(DEFINE_SIMD_STORE_LANE(v128_store32_lane2, simde_wasm_v128_store32_lane, u32, 1)
+)w2c_template"
+R"w2c_template(DEFINE_SIMD_STORE_LANE(v128_store32_lane3, simde_wasm_v128_store32_lane, u32, 0)
+)w2c_template"
+R"w2c_template(DEFINE_SIMD_STORE_LANE(v128_store64_lane0, simde_wasm_v128_store64_lane, u64, 1)
+)w2c_template"
+R"w2c_template(DEFINE_SIMD_STORE_LANE(v128_store64_lane1, simde_wasm_v128_store64_lane, u64, 0)
+)w2c_template"
+R"w2c_template(#else
+)w2c_template"
+R"w2c_template(DEFINE_SIMD_STORE_LANE(v128_store8_lane0, simde_wasm_v128_store8_lane, u8, 0)
 )w2c_template"
 R"w2c_template(DEFINE_SIMD_STORE_LANE(v128_store8_lane1, simde_wasm_v128_store8_lane, u8, 1)
 )w2c_template"
@@ -219,6 +384,279 @@ R"w2c_template(DEFINE_SIMD_STORE_LANE(v128_store64_lane0, simde_wasm_v128_store6
 )w2c_template"
 R"w2c_template(DEFINE_SIMD_STORE_LANE(v128_store64_lane1, simde_wasm_v128_store64_lane, u64, 1)
 )w2c_template"
+R"w2c_template(#endif
+)w2c_template"
+R"w2c_template(
+#if WABT_BIG_ENDIAN
+)w2c_template"
+R"w2c_template(#define v128_const(a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p) simde_wasm_i8x16_const(p,o,n,m,l,k,j,i,h,g,f,e,d,c,b,a)
+)w2c_template"
+R"w2c_template(#define v128_i8x16_extract_lane(v, l) simde_wasm_i8x16_extract_lane(v, 15-(l))
+)w2c_template"
+R"w2c_template(#define v128_u8x16_extract_lane(v, l) simde_wasm_u8x16_extract_lane(v, 15-(l))
+)w2c_template"
+R"w2c_template(#define v128_i16x8_extract_lane(v, l) simde_wasm_i16x8_extract_lane(v, 7-(l))
+)w2c_template"
+R"w2c_template(#define v128_u16x8_extract_lane(v, l) simde_wasm_u16x8_extract_lane(v, 7-(l))
+)w2c_template"
+R"w2c_template(#define v128_i32x4_extract_lane(v, l) simde_wasm_i32x4_extract_lane(v, 3-(l))
+)w2c_template"
+R"w2c_template(#define v128_i64x2_extract_lane(v, l) simde_wasm_i64x2_extract_lane(v, 1-(l))
+)w2c_template"
+R"w2c_template(#define v128_f32x4_extract_lane(v, l) simde_wasm_f32x4_extract_lane(v, 3-(l))
+)w2c_template"
+R"w2c_template(#define v128_f64x2_extract_lane(v, l) simde_wasm_f64x2_extract_lane(v, 1-(l))
+)w2c_template"
+R"w2c_template(#define v128_i8x16_replace_lane(v, l, x) simde_wasm_i8x16_replace_lane(v, 15-(l), x)
+)w2c_template"
+R"w2c_template(#define v128_u8x16_replace_lane(v, l, x) simde_wasm_u8x16_replace_lane(v, 15-(l), x)
+)w2c_template"
+R"w2c_template(#define v128_i16x8_replace_lane(v, l, x) simde_wasm_i16x8_replace_lane(v, 7-(l), x)
+)w2c_template"
+R"w2c_template(#define v128_u16x8_replace_lane(v, l, x) simde_wasm_u16x8_replace_lane(v, 7-(l), x)
+)w2c_template"
+R"w2c_template(#define v128_i32x4_replace_lane(v, l, x) simde_wasm_i32x4_replace_lane(v, 3-(l), x)
+)w2c_template"
+R"w2c_template(#define v128_i64x2_replace_lane(v, l, x) simde_wasm_i64x2_replace_lane(v, 1-(l), x)
+)w2c_template"
+R"w2c_template(#define v128_f32x4_replace_lane(v, l, x) simde_wasm_f32x4_replace_lane(v, 3-(l), x)
+)w2c_template"
+R"w2c_template(#define v128_f64x2_replace_lane(v, l, x) simde_wasm_f64x2_replace_lane(v, 1-(l), x)
+)w2c_template"
+R"w2c_template(#define v128_i8x16_bitmask(v) simde_wasm_i8x16_bitmask(simde_wasm_i8x16_swizzle(v, simde_wasm_i8x16_const(15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0)))
+)w2c_template"
+R"w2c_template(#define v128_i16x8_bitmask(v) simde_wasm_i16x8_bitmask(simde_wasm_i8x16_swizzle(v, simde_wasm_i8x16_const(14,15,12,13,10,11,8,9,6,7,4,5,2,3,0,1)))
+)w2c_template"
+R"w2c_template(#define v128_i32x4_bitmask(v) simde_wasm_i32x4_bitmask(simde_wasm_i8x16_swizzle(v, simde_wasm_i8x16_const(12,13,14,15,8,9,10,11,4,5,6,7,0,1,2,3)))
+)w2c_template"
+R"w2c_template(#define v128_i64x2_bitmask(v) simde_wasm_i64x2_bitmask(simde_wasm_i8x16_swizzle(v, simde_wasm_i8x16_const(8,9,10,11,12,13,14,15,0,1,2,3,4,5,6,7)))
+)w2c_template"
+R"w2c_template(#define v128_i8x16_swizzle(v1, v2) simde_wasm_i8x16_swizzle(v1, simde_wasm_v128_xor(v2, simde_wasm_i8x16_splat(15)))
+)w2c_template"
+R"w2c_template(#define v128_i8x16_shuffle(v1,v2,a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p) simde_wasm_i8x16_shuffle(v2,v1,31-(p),31-(o),31-(n),31-(m),31-(l),31-(k),31-(j),31-(i),31-(h),31-(g),31-(f),31-(e),31-(d),31-(c),31-(b),31-(a))
+)w2c_template"
+R"w2c_template(#define v128_i16x8_extmul_high_i8x16 simde_wasm_i16x8_extmul_low_i8x16
+)w2c_template"
+R"w2c_template(#define v128_u16x8_extmul_high_u8x16 simde_wasm_u16x8_extmul_low_u8x16
+)w2c_template"
+R"w2c_template(#define v128_i16x8_extmul_low_i8x16  simde_wasm_i16x8_extmul_high_i8x16
+)w2c_template"
+R"w2c_template(#define v128_u16x8_extmul_low_u8x16  simde_wasm_u16x8_extmul_high_u8x16
+)w2c_template"
+R"w2c_template(#define v128_i32x4_extmul_high_i16x8 simde_wasm_i32x4_extmul_low_i16x8
+)w2c_template"
+R"w2c_template(#define v128_u32x4_extmul_high_u16x8 simde_wasm_u32x4_extmul_low_u16x8
+)w2c_template"
+R"w2c_template(#define v128_i32x4_extmul_low_i16x8  simde_wasm_i32x4_extmul_high_i16x8
+)w2c_template"
+R"w2c_template(#define v128_u32x4_extmul_low_u16x8  simde_wasm_u32x4_extmul_high_u16x8
+)w2c_template"
+R"w2c_template(#define v128_i64x2_extmul_high_i32x4 simde_wasm_i64x2_extmul_low_i32x4
+)w2c_template"
+R"w2c_template(#define v128_u64x2_extmul_high_u32x4 simde_wasm_u64x2_extmul_low_u32x4
+)w2c_template"
+R"w2c_template(#define v128_i64x2_extmul_low_i32x4  simde_wasm_i64x2_extmul_high_i32x4
+)w2c_template"
+R"w2c_template(#define v128_u64x2_extmul_low_u32x4  simde_wasm_u64x2_extmul_high_u32x4
+)w2c_template"
+R"w2c_template(#define v128_i16x8_extend_high_i8x16 simde_wasm_i16x8_extend_low_i8x16
+)w2c_template"
+R"w2c_template(#define v128_u16x8_extend_high_u8x16 simde_wasm_u16x8_extend_low_u8x16
+)w2c_template"
+R"w2c_template(#define v128_i16x8_extend_low_i8x16  simde_wasm_i16x8_extend_high_i8x16
+)w2c_template"
+R"w2c_template(#define v128_u16x8_extend_low_u8x16  simde_wasm_u16x8_extend_high_u8x16
+)w2c_template"
+R"w2c_template(#define v128_i32x4_extend_high_i16x8 simde_wasm_i32x4_extend_low_i16x8
+)w2c_template"
+R"w2c_template(#define v128_u32x4_extend_high_u16x8 simde_wasm_u32x4_extend_low_u16x8
+)w2c_template"
+R"w2c_template(#define v128_i32x4_extend_low_i16x8  simde_wasm_i32x4_extend_high_i16x8
+)w2c_template"
+R"w2c_template(#define v128_u32x4_extend_low_u16x8  simde_wasm_u32x4_extend_high_u16x8
+)w2c_template"
+R"w2c_template(#define v128_i64x2_extend_high_i32x4 simde_wasm_i64x2_extend_low_i32x4
+)w2c_template"
+R"w2c_template(#define v128_u64x2_extend_high_u32x4 simde_wasm_u64x2_extend_low_u32x4
+)w2c_template"
+R"w2c_template(#define v128_i64x2_extend_low_i32x4  simde_wasm_i64x2_extend_high_i32x4
+)w2c_template"
+R"w2c_template(#define v128_u64x2_extend_low_u32x4  simde_wasm_u64x2_extend_high_u32x4
+)w2c_template"
+R"w2c_template(#define v128_i32x4_trunc_sat_f64x2_zero(a)      \
+)w2c_template"
+R"w2c_template(  simde_wasm_i8x16_swizzle(                     \
+)w2c_template"
+R"w2c_template(      simde_wasm_i32x4_trunc_sat_f64x2_zero(a), \
+)w2c_template"
+R"w2c_template(      simde_wasm_i8x16_const(8,9,10,11,12,13,14,15,0,1,2,3,4,5,6,7))
+)w2c_template"
+R"w2c_template(#define v128_u32x4_trunc_sat_f64x2_zero(a)      \
+)w2c_template"
+R"w2c_template(  simde_wasm_i8x16_swizzle(                     \
+)w2c_template"
+R"w2c_template(      simde_wasm_u32x4_trunc_sat_f64x2_zero(a), \
+)w2c_template"
+R"w2c_template(      simde_wasm_i8x16_const(8,9,10,11,12,13,14,15,0,1,2,3,4,5,6,7))
+)w2c_template"
+R"w2c_template(#define v128_i16x8_narrow_i32x4(a,b) simde_wasm_i16x8_narrow_i32x4(b,a)
+)w2c_template"
+R"w2c_template(#define v128_u16x8_narrow_i32x4(a,b) simde_wasm_u16x8_narrow_i32x4(b,a)
+)w2c_template"
+R"w2c_template(#define v128_i8x16_narrow_i16x8(a,b) simde_wasm_i8x16_narrow_i16x8(b,a)
+)w2c_template"
+R"w2c_template(#define v128_u8x16_narrow_i16x8(a,b) simde_wasm_u8x16_narrow_i16x8(b,a)
+)w2c_template"
+R"w2c_template(#define v128_f64x2_promote_low_f32x4(a)                        \
+)w2c_template"
+R"w2c_template(  simde_wasm_f64x2_promote_low_f32x4(simde_wasm_i8x16_swizzle( \
+)w2c_template"
+R"w2c_template(      a,                                                       \
+)w2c_template"
+R"w2c_template(      simde_wasm_i8x16_const(8,9,10,11,12,13,14,15,0,1,2,3,4,5,6,7)))
+)w2c_template"
+R"w2c_template(#define v128_f32x4_demote_f64x2_zero(a)      \
+)w2c_template"
+R"w2c_template(  simde_wasm_i8x16_swizzle(                  \
+)w2c_template"
+R"w2c_template(      simde_wasm_f32x4_demote_f64x2_zero(a), \
+)w2c_template"
+R"w2c_template(      simde_wasm_i8x16_const(8,9,10,11,12,13,14,15,0,1,2,3,4,5,6,7))
+)w2c_template"
+R"w2c_template(#define v128_f64x2_convert_low_i32x4(a)                        \
+)w2c_template"
+R"w2c_template(  simde_wasm_f64x2_convert_low_i32x4(simde_wasm_i8x16_swizzle( \
+)w2c_template"
+R"w2c_template(      a,                                                       \
+)w2c_template"
+R"w2c_template(      simde_wasm_i8x16_const(8,9,10,11,12,13,14,15,0,1,2,3,4,5,6,7)))
+)w2c_template"
+R"w2c_template(#define v128_f64x2_convert_low_u32x4(a)                        \
+)w2c_template"
+R"w2c_template(  simde_wasm_f64x2_convert_low_u32x4(simde_wasm_i8x16_swizzle( \
+)w2c_template"
+R"w2c_template(      a,                                                       \
+)w2c_template"
+R"w2c_template(      simde_wasm_i8x16_const(8,9,10,11,12,13,14,15,0,1,2,3,4,5,6,7)))
+)w2c_template"
+R"w2c_template(#else
+)w2c_template"
+R"w2c_template(#define v128_const simde_wasm_i8x16_const
+)w2c_template"
+R"w2c_template(#define v128_i8x16_extract_lane simde_wasm_i8x16_extract_lane
+)w2c_template"
+R"w2c_template(#define v128_u8x16_extract_lane simde_wasm_u8x16_extract_lane
+)w2c_template"
+R"w2c_template(#define v128_i16x8_extract_lane simde_wasm_i16x8_extract_lane
+)w2c_template"
+R"w2c_template(#define v128_u16x8_extract_lane simde_wasm_u16x8_extract_lane
+)w2c_template"
+R"w2c_template(#define v128_i32x4_extract_lane simde_wasm_i32x4_extract_lane
+)w2c_template"
+R"w2c_template(#define v128_i64x2_extract_lane simde_wasm_i64x2_extract_lane
+)w2c_template"
+R"w2c_template(#define v128_f32x4_extract_lane simde_wasm_f32x4_extract_lane
+)w2c_template"
+R"w2c_template(#define v128_f64x2_extract_lane simde_wasm_f64x2_extract_lane
+)w2c_template"
+R"w2c_template(#define v128_i8x16_replace_lane simde_wasm_i8x16_replace_lane
+)w2c_template"
+R"w2c_template(#define v128_u8x16_replace_lane simde_wasm_u8x16_replace_lane
+)w2c_template"
+R"w2c_template(#define v128_i16x8_replace_lane simde_wasm_i16x8_replace_lane
+)w2c_template"
+R"w2c_template(#define v128_u16x8_replace_lane simde_wasm_u16x8_replace_lane
+)w2c_template"
+R"w2c_template(#define v128_i32x4_replace_lane simde_wasm_i32x4_replace_lane
+)w2c_template"
+R"w2c_template(#define v128_i64x2_replace_lane simde_wasm_i64x2_replace_lane
+)w2c_template"
+R"w2c_template(#define v128_f32x4_replace_lane simde_wasm_f32x4_replace_lane
+)w2c_template"
+R"w2c_template(#define v128_f64x2_replace_lane simde_wasm_f64x2_replace_lane
+)w2c_template"
+R"w2c_template(#define v128_i8x16_bitmask simde_wasm_i8x16_bitmask
+)w2c_template"
+R"w2c_template(#define v128_i16x8_bitmask simde_wasm_i16x8_bitmask
+)w2c_template"
+R"w2c_template(#define v128_i32x4_bitmask simde_wasm_i32x4_bitmask
+)w2c_template"
+R"w2c_template(#define v128_i64x2_bitmask simde_wasm_i64x2_bitmask
+)w2c_template"
+R"w2c_template(#define v128_i8x16_swizzle simde_wasm_i8x16_swizzle
+)w2c_template"
+R"w2c_template(#define v128_i8x16_shuffle simde_wasm_i8x16_shuffle
+)w2c_template"
+R"w2c_template(#define v128_i16x8_extmul_high_i8x16 simde_wasm_i16x8_extmul_high_i8x16
+)w2c_template"
+R"w2c_template(#define v128_u16x8_extmul_high_u8x16 simde_wasm_u16x8_extmul_high_u8x16
+)w2c_template"
+R"w2c_template(#define v128_i16x8_extmul_low_i8x16  simde_wasm_i16x8_extmul_low_i8x16
+)w2c_template"
+R"w2c_template(#define v128_u16x8_extmul_low_u8x16  simde_wasm_u16x8_extmul_low_u8x16
+)w2c_template"
+R"w2c_template(#define v128_i32x4_extmul_high_i16x8 simde_wasm_i32x4_extmul_high_i16x8
+)w2c_template"
+R"w2c_template(#define v128_u32x4_extmul_high_u16x8 simde_wasm_u32x4_extmul_high_u16x8
+)w2c_template"
+R"w2c_template(#define v128_i32x4_extmul_low_i16x8  simde_wasm_i32x4_extmul_low_i16x8
+)w2c_template"
+R"w2c_template(#define v128_u32x4_extmul_low_u16x8  simde_wasm_u32x4_extmul_low_u16x8
+)w2c_template"
+R"w2c_template(#define v128_i64x2_extmul_high_i32x4 simde_wasm_i64x2_extmul_high_i32x4
+)w2c_template"
+R"w2c_template(#define v128_u64x2_extmul_high_u32x4 simde_wasm_u64x2_extmul_high_u32x4
+)w2c_template"
+R"w2c_template(#define v128_i64x2_extmul_low_i32x4  simde_wasm_i64x2_extmul_low_i32x4
+)w2c_template"
+R"w2c_template(#define v128_u64x2_extmul_low_u32x4  simde_wasm_u64x2_extmul_low_u32x4
+)w2c_template"
+R"w2c_template(#define v128_i16x8_extend_high_i8x16 simde_wasm_i16x8_extend_high_i8x16
+)w2c_template"
+R"w2c_template(#define v128_u16x8_extend_high_u8x16 simde_wasm_u16x8_extend_high_u8x16
+)w2c_template"
+R"w2c_template(#define v128_i16x8_extend_low_i8x16  simde_wasm_i16x8_extend_low_i8x16
+)w2c_template"
+R"w2c_template(#define v128_u16x8_extend_low_u8x16  simde_wasm_u16x8_extend_low_u8x16
+)w2c_template"
+R"w2c_template(#define v128_i32x4_extend_high_i16x8 simde_wasm_i32x4_extend_high_i16x8
+)w2c_template"
+R"w2c_template(#define v128_u32x4_extend_high_u16x8 simde_wasm_u32x4_extend_high_u16x8
+)w2c_template"
+R"w2c_template(#define v128_i32x4_extend_low_i16x8  simde_wasm_i32x4_extend_low_i16x8
+)w2c_template"
+R"w2c_template(#define v128_u32x4_extend_low_u16x8  simde_wasm_u32x4_extend_low_u16x8
+)w2c_template"
+R"w2c_template(#define v128_i64x2_extend_high_i32x4 simde_wasm_i64x2_extend_high_i32x4
+)w2c_template"
+R"w2c_template(#define v128_u64x2_extend_high_u32x4 simde_wasm_u64x2_extend_high_u32x4
+)w2c_template"
+R"w2c_template(#define v128_i64x2_extend_low_i32x4  simde_wasm_i64x2_extend_low_i32x4
+)w2c_template"
+R"w2c_template(#define v128_u64x2_extend_low_u32x4  simde_wasm_u64x2_extend_low_u32x4
+)w2c_template"
+R"w2c_template(#define v128_i32x4_trunc_sat_f64x2_zero simde_wasm_i32x4_trunc_sat_f64x2_zero
+)w2c_template"
+R"w2c_template(#define v128_u32x4_trunc_sat_f64x2_zero simde_wasm_u32x4_trunc_sat_f64x2_zero
+)w2c_template"
+R"w2c_template(#define v128_i16x8_narrow_i32x4 simde_wasm_i16x8_narrow_i32x4
+)w2c_template"
+R"w2c_template(#define v128_u16x8_narrow_i32x4 simde_wasm_u16x8_narrow_i32x4
+)w2c_template"
+R"w2c_template(#define v128_i8x16_narrow_i16x8 simde_wasm_i8x16_narrow_i16x8
+)w2c_template"
+R"w2c_template(#define v128_u8x16_narrow_i16x8 simde_wasm_u8x16_narrow_i16x8
+)w2c_template"
+R"w2c_template(#define v128_f64x2_promote_low_f32x4 simde_wasm_f64x2_promote_low_f32x4
+)w2c_template"
+R"w2c_template(#define v128_f32x4_demote_f64x2_zero simde_wasm_f32x4_demote_f64x2_zero
+)w2c_template"
+R"w2c_template(#define v128_f64x2_convert_low_i32x4 simde_wasm_f64x2_convert_low_i32x4
+)w2c_template"
+R"w2c_template(#define v128_f64x2_convert_low_u32x4 simde_wasm_f64x2_convert_low_u32x4
+)w2c_template"
+R"w2c_template(#endif
+)w2c_template"
 R"w2c_template(// clang-format on
 )w2c_template"
 ;

third_party/wasm2c/src/prebuilt/wasm2c_source_declarations.cc

@@ -1,8 +1,119 @@
 const char* s_source_declarations = R"w2c_template(
+// Computes a pointer to an object of the given size in a little-endian memory.
+)w2c_template"
+R"w2c_template(//
+)w2c_template"
+R"w2c_template(// On a little-endian host, this is just &mem->data[addr] - the object's size is
+)w2c_template"
+R"w2c_template(// unused. On a big-endian host, it's &mem->data[mem->size - addr - n], where n
+)w2c_template"
+R"w2c_template(// is the object's size.
+)w2c_template"
+R"w2c_template(//
+)w2c_template"
+R"w2c_template(// Note that mem may be evaluated multiple times.
+)w2c_template"
+R"w2c_template(//
+)w2c_template"
+R"w2c_template(// Parameters:
+)w2c_template"
+R"w2c_template(// mem - The memory.
+)w2c_template"
+R"w2c_template(// addr - The address.
+)w2c_template"
+R"w2c_template(// n - The size of the object.
+)w2c_template"
+R"w2c_template(//
+)w2c_template"
+R"w2c_template(// Result:
+)w2c_template"
+R"w2c_template(// A pointer for an object of size n.
+)w2c_template"
+R"w2c_template(#if WABT_BIG_ENDIAN
+)w2c_template"
+R"w2c_template(#define MEM_ADDR(mem, addr, n) &(mem)->data[(mem)->size - (addr) - (n)]
+)w2c_template"
+R"w2c_template(#else
+)w2c_template"
+R"w2c_template(#define MEM_ADDR(mem, addr, n) &(mem)->data[addr]
+)w2c_template"
+R"w2c_template(#endif
+)w2c_template"
+R"w2c_template(
+// We can only use Segue for this module if it uses a single unshared imported
+)w2c_template"
+R"w2c_template(// or exported memory
+)w2c_template"
+R"w2c_template(#if WASM_RT_USE_SEGUE && IS_SINGLE_UNSHARED_MEMORY
+)w2c_template"
+R"w2c_template(#define WASM_RT_USE_SEGUE_FOR_THIS_MODULE 1
+)w2c_template"
+R"w2c_template(#else
+)w2c_template"
+R"w2c_template(#define WASM_RT_USE_SEGUE_FOR_THIS_MODULE 0
+)w2c_template"
+R"w2c_template(#endif
+)w2c_template"
+R"w2c_template(
+#if WASM_RT_USE_SEGUE_FOR_THIS_MODULE
+)w2c_template"
+R"w2c_template(// POSIX uses FS for TLS, GS is free
+)w2c_template"
+R"w2c_template(static inline void* wasm_rt_segue_read_base() {
+)w2c_template"
+R"w2c_template(  if (wasm_rt_fsgsbase_inst_supported) {
+)w2c_template"
+R"w2c_template(    return (void*)__builtin_ia32_rdgsbase64();
+)w2c_template"
+R"w2c_template(  } else {
+)w2c_template"
+R"w2c_template(    return wasm_rt_syscall_get_segue_base();
+)w2c_template"
+R"w2c_template(  }
+)w2c_template"
+R"w2c_template(}
+)w2c_template"
+R"w2c_template(static inline void wasm_rt_segue_write_base(void* base) {
+)w2c_template"
+R"w2c_template(#if WASM_RT_SEGUE_FREE_SEGMENT
+)w2c_template"
+R"w2c_template(  if (wasm_rt_last_segment_val == base) {
+)w2c_template"
+R"w2c_template(    return;
+)w2c_template"
+R"w2c_template(  }
+)w2c_template"
+R"w2c_template(
+  wasm_rt_last_segment_val = base;
+)w2c_template"
+R"w2c_template(#endif
+)w2c_template"
+R"w2c_template(
+  if (wasm_rt_fsgsbase_inst_supported) {
+)w2c_template"
+R"w2c_template(    __builtin_ia32_wrgsbase64((uintptr_t)base);
+)w2c_template"
+R"w2c_template(  } else {
+)w2c_template"
+R"w2c_template(    wasm_rt_syscall_set_segue_base(base);
+)w2c_template"
+R"w2c_template(  }
+)w2c_template"
+R"w2c_template(}
+)w2c_template"
+R"w2c_template(#define MEM_ADDR_MEMOP(mem, addr, n) ((uint8_t __seg_gs*)(uintptr_t)addr)
+)w2c_template"
+R"w2c_template(#else
+)w2c_template"
+R"w2c_template(#define MEM_ADDR_MEMOP(mem, addr, n) MEM_ADDR(mem, addr, n)
+)w2c_template"
+R"w2c_template(#endif
+)w2c_template"
+R"w2c_template(
 #define TRAP(x) (wasm_rt_trap(WASM_RT_TRAP_##x), 0)
 )w2c_template"
 R"w2c_template(
-#if WASM_RT_USE_STACK_DEPTH_COUNT
+#if WASM_RT_STACK_DEPTH_COUNT
 )w2c_template"
 R"w2c_template(#define FUNC_PROLOGUE                                            \
 )w2c_template"
@@ -35,15 +146,23 @@ R"w2c_template(  return (a == b) || LIKELY(a && b && !memcmp(a, b, 32));
 R"w2c_template(}
 )w2c_template"
 R"w2c_template(
-#define CALL_INDIRECT(table, t, ft, x, ...)              \
+#define CHECK_CALL_INDIRECT(table, ft, x)                \
 )w2c_template"
 R"w2c_template(  (LIKELY((x) < table.size && table.data[x].func &&      \
 )w2c_template"
 R"w2c_template(          func_types_eq(ft, table.data[x].func_type)) || \
 )w2c_template"
-R"w2c_template(       TRAP(CALL_INDIRECT),                              \
+R"w2c_template(   TRAP(CALL_INDIRECT))
 )w2c_template"
-R"w2c_template(   ((t)table.data[x].func)(__VA_ARGS__))
+R"w2c_template(
+#define DO_CALL_INDIRECT(table, t, x, ...) ((t)table.data[x].func)(__VA_ARGS__)
+)w2c_template"
+R"w2c_template(
+#define CALL_INDIRECT(table, t, ft, x, ...) \
+)w2c_template"
+R"w2c_template(  (CHECK_CALL_INDIRECT(table, ft, x),       \
+)w2c_template"
+R"w2c_template(   DO_CALL_INDIRECT(table, t, x, __VA_ARGS__))
 )w2c_template"
 R"w2c_template(
 #ifdef SUPPORT_MEMORY64
@@ -75,31 +194,72 @@ R"w2c_template(    TRAP(OOB);
 R"w2c_template(#endif
 )w2c_template"
 R"w2c_template(
-#if WASM_RT_MEMCHECK_GUARD_PAGES
+#if WASM_RT_USE_SEGUE_FOR_THIS_MODULE && WASM_RT_SANITY_CHECKS
 )w2c_template"
-R"w2c_template(#define MEMCHECK(mem, a, t)
+R"w2c_template(#include <stdio.h>
+)w2c_template"
+R"w2c_template(#define WASM_RT_CHECK_BASE(mem)                                               \
+)w2c_template"
+R"w2c_template(  if (((uintptr_t)((mem)->data)) != ((uintptr_t)wasm_rt_segue_read_base())) { \
+)w2c_template"
+R"w2c_template(    puts("Segment register mismatch\n");                                      \
+)w2c_template"
+R"w2c_template(    abort();                                                                  \
+)w2c_template"
+R"w2c_template(  }
 )w2c_template"
 R"w2c_template(#else
 )w2c_template"
-R"w2c_template(#define MEMCHECK(mem, a, t) RANGE_CHECK(mem, a, sizeof(t))
+R"w2c_template(#define WASM_RT_CHECK_BASE(mem)
+)w2c_template"
+R"w2c_template(#endif
+)w2c_template"
+R"w2c_template(
+#if WASM_RT_MEMCHECK_GUARD_PAGES
+)w2c_template"
+R"w2c_template(#define MEMCHECK(mem, a, t) WASM_RT_CHECK_BASE(mem);
+)w2c_template"
+R"w2c_template(#else
+)w2c_template"
+R"w2c_template(#define MEMCHECK(mem, a, t) \
+)w2c_template"
+R"w2c_template(  WASM_RT_CHECK_BASE(mem);  \
+)w2c_template"
+R"w2c_template(  RANGE_CHECK(mem, a, sizeof(t))
 )w2c_template"
 R"w2c_template(#endif
 )w2c_template"
 R"w2c_template(
 #ifdef __GNUC__
 )w2c_template"
-R"w2c_template(#define wasm_asm __asm__
+R"w2c_template(#define FORCE_READ_INT(var) __asm__("" ::"r"(var));
+)w2c_template"
+R"w2c_template(// Clang on Mips requires "f" constraints on floats
+)w2c_template"
+R"w2c_template(// See https://github.com/llvm/llvm-project/issues/64241
+)w2c_template"
+R"w2c_template(#if defined(__clang__) && \
+)w2c_template"
+R"w2c_template(    (defined(mips) || defined(__mips__) || defined(__mips))
+)w2c_template"
+R"w2c_template(#define FORCE_READ_FLOAT(var) __asm__("" ::"f"(var));
 )w2c_template"
 R"w2c_template(#else
 )w2c_template"
-R"w2c_template(#define wasm_asm(X)
+R"w2c_template(#define FORCE_READ_FLOAT(var) __asm__("" ::"r"(var));
+)w2c_template"
+R"w2c_template(#endif
+)w2c_template"
+R"w2c_template(#else
+)w2c_template"
+R"w2c_template(#define FORCE_READ_INT(var)
+)w2c_template"
+R"w2c_template(#define FORCE_READ_FLOAT(var)
 )w2c_template"
 R"w2c_template(#endif
 )w2c_template"
 R"w2c_template(
-#if WABT_BIG_ENDIAN
+static inline void load_data(void* dest, const void* src, size_t n) {
-)w2c_template"
-R"w2c_template(static inline void load_data(void* dest, const void* src, size_t n) {
 )w2c_template"
 R"w2c_template(  if (!n) {
 )w2c_template"
@@ -107,13 +267,13 @@ R"w2c_template(    return;
 )w2c_template"
 R"w2c_template(  }
 )w2c_template"
-R"w2c_template(  size_t i = 0;
+R"w2c_template(  wasm_rt_memcpy(dest, src, n);
+)w2c_template"
+R"w2c_template(#if WABT_BIG_ENDIAN
 )w2c_template"
 R"w2c_template(  u8* dest_chars = dest;
 )w2c_template"
-R"w2c_template(  memcpy(dest, src, n);
+R"w2c_template(  for (size_t i = 0; i < (n >> 1); i++) {
-)w2c_template"
-R"w2c_template(  for (i = 0; i < (n >> 1); i++) {
 )w2c_template"
 R"w2c_template(    u8 cursor = dest_chars[i];
 )w2c_template"
@@ -123,88 +283,37 @@ R"w2c_template(    dest_chars[n - i - 1] = cursor;
 )w2c_template"
 R"w2c_template(  }
 )w2c_template"
+R"w2c_template(#endif
+)w2c_template"
 R"w2c_template(}
 )w2c_template"
-R"w2c_template(#define LOAD_DATA(m, o, i, s)                   \
-)w2c_template"
-R"w2c_template(  do {                                          \
-)w2c_template"
-R"w2c_template(    RANGE_CHECK((&m), m.size - o - s, s);       \
-)w2c_template"
-R"w2c_template(    load_data(&(m.data[m.size - o - s]), i, s); \
-)w2c_template"
-R"w2c_template(  } while (0)
-)w2c_template"
-R"w2c_template(#define DEFINE_LOAD(name, t1, t2, t3)                                  \
-)w2c_template"
-R"w2c_template(  static inline t3 name(wasm_rt_memory_t* mem, u64 addr) {             \
-)w2c_template"
-R"w2c_template(    MEMCHECK(mem, addr, t1);                                           \
-)w2c_template"
-R"w2c_template(    t1 result;                                                         \
-)w2c_template"
-R"w2c_template(    wasm_rt_memcpy(&result, &mem->data[mem->size - addr - sizeof(t1)], \
-)w2c_template"
-R"w2c_template(                   sizeof(t1));                                        \
-)w2c_template"
-R"w2c_template(    wasm_asm("" ::"r"(result));                                        \
-)w2c_template"
-R"w2c_template(    return (t3)(t2)result;                                             \
-)w2c_template"
-R"w2c_template(  }
-)w2c_template"
 R"w2c_template(
-#define DEFINE_STORE(name, t1, t2)                                      \
+#define LOAD_DATA(m, o, i, s)            \
 )w2c_template"
-R"w2c_template(  static inline void name(wasm_rt_memory_t* mem, u64 addr, t2 value) {  \
+R"w2c_template(  do {                                   \
 )w2c_template"
-R"w2c_template(    MEMCHECK(mem, addr, t1);                                            \
+R"w2c_template(    RANGE_CHECK((&m), o, s);             \
 )w2c_template"
-R"w2c_template(    t1 wrapped = (t1)value;                                             \
+R"w2c_template(    load_data(MEM_ADDR(&m, o, s), i, s); \
-)w2c_template"
-R"w2c_template(    wasm_rt_memcpy(&mem->data[mem->size - addr - sizeof(t1)], &wrapped, \
-)w2c_template"
-R"w2c_template(                   sizeof(t1));                                         \
-)w2c_template"
-R"w2c_template(  }
-)w2c_template"
-R"w2c_template(#else
-)w2c_template"
-R"w2c_template(static inline void load_data(void* dest, const void* src, size_t n) {
-)w2c_template"
-R"w2c_template(  if (!n) {
-)w2c_template"
-R"w2c_template(    return;
-)w2c_template"
-R"w2c_template(  }
-)w2c_template"
-R"w2c_template(  memcpy(dest, src, n);
-)w2c_template"
-R"w2c_template(}
-)w2c_template"
-R"w2c_template(#define LOAD_DATA(m, o, i, s)      \
-)w2c_template"
-R"w2c_template(  do {                             \
-)w2c_template"
-R"w2c_template(    RANGE_CHECK((&m), o, s);       \
-)w2c_template"
-R"w2c_template(    load_data(&(m.data[o]), i, s); \
 )w2c_template"
 R"w2c_template(  } while (0)
 )w2c_template"
-R"w2c_template(#define DEFINE_LOAD(name, t1, t2, t3)                      \
+R"w2c_template(
+#define DEFINE_LOAD(name, t1, t2, t3, force_read)                  \
 )w2c_template"
-R"w2c_template(  static inline t3 name(wasm_rt_memory_t* mem, u64 addr) { \
+R"w2c_template(  static inline t3 name(wasm_rt_memory_t* mem, u64 addr) {         \
 )w2c_template"
-R"w2c_template(    MEMCHECK(mem, addr, t1);                               \
+R"w2c_template(    MEMCHECK(mem, addr, t1);                                       \
 )w2c_template"
-R"w2c_template(    t1 result;                                             \
+R"w2c_template(    t1 result;                                                     \
 )w2c_template"
-R"w2c_template(    wasm_rt_memcpy(&result, &mem->data[addr], sizeof(t1)); \
+R"w2c_template(    wasm_rt_memcpy(&result, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), \
 )w2c_template"
-R"w2c_template(    wasm_asm("" ::"r"(result));                            \
+R"w2c_template(                   sizeof(t1));                                    \
 )w2c_template"
-R"w2c_template(    return (t3)(t2)result;                                 \
+R"w2c_template(    force_read(result);                                            \
+)w2c_template"
+R"w2c_template(    return (t3)(t2)result;                                         \
 )w2c_template"
 R"w2c_template(  }
 )w2c_template"
@@ -217,40 +326,40 @@ R"w2c_template(    MEMCHECK(mem, addr, t1);
 )w2c_template"
 R"w2c_template(    t1 wrapped = (t1)value;                                            \
 )w2c_template"
-R"w2c_template(    wasm_rt_memcpy(&mem->data[addr], &wrapped, sizeof(t1));            \
+R"w2c_template(    wasm_rt_memcpy(MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), &wrapped,    \
+)w2c_template"
+R"w2c_template(                   sizeof(t1));                                        \
 )w2c_template"
 R"w2c_template(  }
 )w2c_template"
-R"w2c_template(#endif
-)w2c_template"
 R"w2c_template(
-DEFINE_LOAD(i32_load, u32, u32, u32)
+DEFINE_LOAD(i32_load, u32, u32, u32, FORCE_READ_INT)
 )w2c_template"
-R"w2c_template(DEFINE_LOAD(i64_load, u64, u64, u64)
+R"w2c_template(DEFINE_LOAD(i64_load, u64, u64, u64, FORCE_READ_INT)
 )w2c_template"
-R"w2c_template(DEFINE_LOAD(f32_load, f32, f32, f32)
+R"w2c_template(DEFINE_LOAD(f32_load, f32, f32, f32, FORCE_READ_FLOAT)
 )w2c_template"
-R"w2c_template(DEFINE_LOAD(f64_load, f64, f64, f64)
+R"w2c_template(DEFINE_LOAD(f64_load, f64, f64, f64, FORCE_READ_FLOAT)
 )w2c_template"
-R"w2c_template(DEFINE_LOAD(i32_load8_s, s8, s32, u32)
+R"w2c_template(DEFINE_LOAD(i32_load8_s, s8, s32, u32, FORCE_READ_INT)
 )w2c_template"
-R"w2c_template(DEFINE_LOAD(i64_load8_s, s8, s64, u64)
+R"w2c_template(DEFINE_LOAD(i64_load8_s, s8, s64, u64, FORCE_READ_INT)
 )w2c_template"
-R"w2c_template(DEFINE_LOAD(i32_load8_u, u8, u32, u32)
+R"w2c_template(DEFINE_LOAD(i32_load8_u, u8, u32, u32, FORCE_READ_INT)
 )w2c_template"
-R"w2c_template(DEFINE_LOAD(i64_load8_u, u8, u64, u64)
+R"w2c_template(DEFINE_LOAD(i64_load8_u, u8, u64, u64, FORCE_READ_INT)
 )w2c_template"
-R"w2c_template(DEFINE_LOAD(i32_load16_s, s16, s32, u32)
+R"w2c_template(DEFINE_LOAD(i32_load16_s, s16, s32, u32, FORCE_READ_INT)
 )w2c_template"
-R"w2c_template(DEFINE_LOAD(i64_load16_s, s16, s64, u64)
+R"w2c_template(DEFINE_LOAD(i64_load16_s, s16, s64, u64, FORCE_READ_INT)
 )w2c_template"
-R"w2c_template(DEFINE_LOAD(i32_load16_u, u16, u32, u32)
+R"w2c_template(DEFINE_LOAD(i32_load16_u, u16, u32, u32, FORCE_READ_INT)
 )w2c_template"
-R"w2c_template(DEFINE_LOAD(i64_load16_u, u16, u64, u64)
+R"w2c_template(DEFINE_LOAD(i64_load16_u, u16, u64, u64, FORCE_READ_INT)
 )w2c_template"
-R"w2c_template(DEFINE_LOAD(i64_load32_s, s32, s64, u64)
+R"w2c_template(DEFINE_LOAD(i64_load32_s, s32, s64, u64, FORCE_READ_INT)
 )w2c_template"
-R"w2c_template(DEFINE_LOAD(i64_load32_u, u32, u64, u64)
+R"w2c_template(DEFINE_LOAD(i64_load32_u, u32, u64, u64, FORCE_READ_INT)
 )w2c_template"
 R"w2c_template(DEFINE_STORE(i32_store, u32, u32)
 )w2c_template"
@@ -617,15 +726,15 @@ R"w2c_template(#define I64_TRUNC_SAT_U_F64(x) \
 R"w2c_template(  TRUNC_SAT_U(u64, f64, (f64)UINT64_MAX, UINT64_MAX, x)
 )w2c_template"
 R"w2c_template(
-#define DEFINE_REINTERPRET(name, t1, t2) \
+#define DEFINE_REINTERPRET(name, t1, t2)         \
 )w2c_template"
-R"w2c_template(  static inline t2 name(t1 x) {          \
+R"w2c_template(  static inline t2 name(t1 x) {                  \
 )w2c_template"
-R"w2c_template(    t2 result;                           \
+R"w2c_template(    t2 result;                                   \
 )w2c_template"
-R"w2c_template(    memcpy(&result, &x, sizeof(result)); \
+R"w2c_template(    wasm_rt_memcpy(&result, &x, sizeof(result)); \
 )w2c_template"
-R"w2c_template(    return result;                       \
+R"w2c_template(    return result;                               \
 )w2c_template"
 R"w2c_template(  }
 )w2c_template"
@@ -643,11 +752,11 @@ static float quiet_nanf(float x) {
 )w2c_template"
 R"w2c_template(  uint32_t tmp;
 )w2c_template"
-R"w2c_template(  memcpy(&tmp, &x, 4);
+R"w2c_template(  wasm_rt_memcpy(&tmp, &x, 4);
 )w2c_template"
 R"w2c_template(  tmp |= 0x7fc00000lu;
 )w2c_template"
-R"w2c_template(  memcpy(&x, &tmp, 4);
+R"w2c_template(  wasm_rt_memcpy(&x, &tmp, 4);
 )w2c_template"
 R"w2c_template(  return x;
 )w2c_template"
@@ -658,11 +767,11 @@ static double quiet_nan(double x) {
 )w2c_template"
 R"w2c_template(  uint64_t tmp;
 )w2c_template"
-R"w2c_template(  memcpy(&tmp, &x, 8);
+R"w2c_template(  wasm_rt_memcpy(&tmp, &x, 8);
 )w2c_template"
 R"w2c_template(  tmp |= 0x7ff8000000000000llu;
 )w2c_template"
-R"w2c_template(  memcpy(&x, &tmp, 8);
+R"w2c_template(  wasm_rt_memcpy(&x, &tmp, 8);
 )w2c_template"
 R"w2c_template(  return x;
 )w2c_template"
@@ -805,11 +914,11 @@ R"w2c_template(  if (UNLIKELY(isnan(x))) {
 )w2c_template"
 R"w2c_template(    uint32_t tmp;
 )w2c_template"
-R"w2c_template(    memcpy(&tmp, &x, 4);
+R"w2c_template(    wasm_rt_memcpy(&tmp, &x, 4);
 )w2c_template"
 R"w2c_template(    tmp = tmp & ~(1UL << 31);
 )w2c_template"
-R"w2c_template(    memcpy(&x, &tmp, 4);
+R"w2c_template(    wasm_rt_memcpy(&x, &tmp, 4);
 )w2c_template"
 R"w2c_template(    return x;
 )w2c_template"
@@ -826,11 +935,11 @@ R"w2c_template(  if (UNLIKELY(isnan(x))) {
 )w2c_template"
 R"w2c_template(    uint64_t tmp;
 )w2c_template"
-R"w2c_template(    memcpy(&tmp, &x, 8);
+R"w2c_template(    wasm_rt_memcpy(&tmp, &x, 8);
 )w2c_template"
 R"w2c_template(    tmp = tmp & ~(1ULL << 63);
 )w2c_template"
-R"w2c_template(    memcpy(&x, &tmp, 8);
+R"w2c_template(    wasm_rt_memcpy(&x, &tmp, 8);
 )w2c_template"
 R"w2c_template(    return x;
 )w2c_template"
@@ -871,7 +980,7 @@ static inline void memory_fill(wasm_rt_memory_t* mem, u32 d, u32 val, u32 n) {
 )w2c_template"
 R"w2c_template(  RANGE_CHECK(mem, d, n);
 )w2c_template"
-R"w2c_template(  memset(mem->data + d, val, n);
+R"w2c_template(  memset(MEM_ADDR(mem, d, n), val, n);
 )w2c_template"
 R"w2c_template(}
 )w2c_template"
@@ -890,7 +999,7 @@ R"w2c_template(  RANGE_CHECK(dest, dest_addr, n);
 )w2c_template"
 R"w2c_template(  RANGE_CHECK(src, src_addr, n);
 )w2c_template"
-R"w2c_template(  memmove(dest->data + dest_addr, src->data + src_addr, n);
+R"w2c_template(  memmove(MEM_ADDR(dest, dest_addr, n), MEM_ADDR(src, src_addr, n), n);
 )w2c_template"
 R"w2c_template(}
 )w2c_template"
@@ -918,10 +1027,14 @@ R"w2c_template(}
 R"w2c_template(
 typedef struct {
 )w2c_template"
+R"w2c_template(  enum { RefFunc, RefNull, GlobalGet } expr_type;
+)w2c_template"
 R"w2c_template(  wasm_rt_func_type_t type;
 )w2c_template"
 R"w2c_template(  wasm_rt_function_ptr_t func;
 )w2c_template"
+R"w2c_template(  wasm_rt_tailcallee_t func_tailcallee;
+)w2c_template"
 R"w2c_template(  size_t module_offset;
 )w2c_template"
 R"w2c_template(} wasm_elem_segment_expr_t;
@@ -951,20 +1064,44 @@ R"w2c_template(    TRAP(OOB);
 )w2c_template"
 R"w2c_template(  for (u32 i = 0; i < n; i++) {
 )w2c_template"
-R"w2c_template(    const wasm_elem_segment_expr_t* src_expr = &src[src_addr + i];
+R"w2c_template(    const wasm_elem_segment_expr_t* const src_expr = &src[src_addr + i];
 )w2c_template"
-R"w2c_template(    dest->data[dest_addr + i] =
+R"w2c_template(    wasm_rt_funcref_t* const dest_val = &(dest->data[dest_addr + i]);
 )w2c_template"
-R"w2c_template(        (wasm_rt_funcref_t){src_expr->type, src_expr->func,
+R"w2c_template(    switch (src_expr->expr_type) {
 )w2c_template"
-R"w2c_template(                            (char*)module_instance + src_expr->module_offset};
+R"w2c_template(      case RefFunc:
+)w2c_template"
+R"w2c_template(        *dest_val = (wasm_rt_funcref_t){
+)w2c_template"
+R"w2c_template(            src_expr->type, src_expr->func, src_expr->func_tailcallee,
+)w2c_template"
+R"w2c_template(            (char*)module_instance + src_expr->module_offset};
+)w2c_template"
+R"w2c_template(        break;
+)w2c_template"
+R"w2c_template(      case RefNull:
+)w2c_template"
+R"w2c_template(        *dest_val = wasm_rt_funcref_null_value;
+)w2c_template"
+R"w2c_template(        break;
+)w2c_template"
+R"w2c_template(      case GlobalGet:
+)w2c_template"
+R"w2c_template(        *dest_val = **(wasm_rt_funcref_t**)((char*)module_instance +
+)w2c_template"
+R"w2c_template(                                            src_expr->module_offset);
+)w2c_template"
+R"w2c_template(        break;
+)w2c_template"
+R"w2c_template(    }
 )w2c_template"
 R"w2c_template(  }
 )w2c_template"
 R"w2c_template(}
 )w2c_template"
 R"w2c_template(
-// Currently Wasm only supports initializing externref tables with ref.null.
+// Currently wasm2c only supports initializing externref tables with ref.null.
 )w2c_template"
 R"w2c_template(static inline void externref_table_init(wasm_rt_externref_table_t* dest,
 )w2c_template"
@@ -1107,4 +1244,38 @@ R"w2c_template(#define FUNC_TYPE_T(x) static const char x[]
 )w2c_template"
 R"w2c_template(#endif
 )w2c_template"
+R"w2c_template(
+#if (__STDC_VERSION__ < 201112L) && !defined(static_assert)
+)w2c_template"
+R"w2c_template(#define static_assert(X) \
+)w2c_template"
+R"w2c_template(  extern int(*assertion(void))[!!sizeof(struct { int x : (X) ? 2 : -1; })];
+)w2c_template"
+R"w2c_template(#endif
+)w2c_template"
+R"w2c_template(
+#ifdef _MSC_VER
+)w2c_template"
+R"w2c_template(#define WEAK_FUNC_DECL(func, fallback)                             \
+)w2c_template"
+R"w2c_template(  __pragma(comment(linker, "/alternatename:" #func "=" #fallback)) \
+)w2c_template"
+R"w2c_template(                                                                   \
+)w2c_template"
+R"w2c_template(      void                                                         \
+)w2c_template"
+R"w2c_template(      fallback(void** instance_ptr, void* tail_call_stack,         \
+)w2c_template"
+R"w2c_template(               wasm_rt_tailcallee_t* next)
+)w2c_template"
+R"w2c_template(#else
+)w2c_template"
+R"w2c_template(#define WEAK_FUNC_DECL(func, fallback)                                        \
+)w2c_template"
+R"w2c_template(  __attribute__((weak)) void func(void** instance_ptr, void* tail_call_stack, \
+)w2c_template"
+R"w2c_template(                                  wasm_rt_tailcallee_t* next)
+)w2c_template"
+R"w2c_template(#endif
+)w2c_template"
 ;

third_party/wasm2c/src/resolve-names.cc

@@ -348,8 +348,8 @@ Result NameResolver::OnLocalTeeExpr(LocalTeeExpr* expr) {
 }
 
 Result NameResolver::OnMemoryCopyExpr(MemoryCopyExpr* expr) {
-  ResolveMemoryVar(&expr->srcmemidx);
   ResolveMemoryVar(&expr->destmemidx);
+  ResolveMemoryVar(&expr->srcmemidx);
   return Result::Ok;
 }
 
@@ -488,12 +488,12 @@ void NameResolver::VisitFunc(Func* func) {
     ResolveFuncTypeVar(&func->decl.type_var);
   }
 
-  func->bindings.FindDuplicates(
+  func->bindings.FindDuplicates([func, this](const BindingHash::value_type& a,
-      [=](const BindingHash::value_type& a, const BindingHash::value_type& b) {
+                                             const BindingHash::value_type& b) {
-        const char* desc =
+    const char* desc =
-            (a.second.index < func->GetNumParams()) ? "parameter" : "local";
+        (a.second.index < func->GetNumParams()) ? "parameter" : "local";
-        PrintDuplicateBindingsError(a, b, desc);
+    PrintDuplicateBindingsError(a, b, desc);
-      });
+  });
 
   visitor_.VisitFunc(func);
   current_func_ = nullptr;

third_party/wasm2c/src/shared-validator.cc

@@ -134,13 +134,25 @@ Result SharedValidator::OnTable(const Location& loc,
   return result;
 }
 
-Result SharedValidator::OnMemory(const Location& loc, const Limits& limits) {
+Result SharedValidator::OnMemory(const Location& loc,
+                                 const Limits& limits,
+                                 uint32_t page_size) {
   Result result = Result::Ok;
   if (memories_.size() > 0 && !options_.features.multi_memory_enabled()) {
     result |= PrintError(loc, "only one memory block allowed");
   }
-  result |= CheckLimits(
+
-      loc, limits, limits.is_64 ? WABT_MAX_PAGES64 : WABT_MAX_PAGES32, "pages");
+  if (page_size != WABT_DEFAULT_PAGE_SIZE) {
+    if (!options_.features.custom_page_sizes_enabled()) {
+      result |= PrintError(loc, "only default page size (64 KiB) is allowed");
+    } else if (page_size != 1) {
+      result |= PrintError(loc, "only page sizes of 1 B or 64 KiB are allowed");
+    }
+  }
+
+  uint64_t absolute_max = WABT_BYTES_TO_MIN_PAGES(
+      (limits.is_64 ? UINT64_MAX : UINT32_MAX), page_size);
+  result |= CheckLimits(loc, limits, absolute_max, "pages");
 
   if (limits.is_shared) {
     if (!options_.features.threads_enabled()) {
@@ -276,27 +288,6 @@ Result SharedValidator::OnElemSegmentElemType(const Location& loc,
   return result;
 }
 
-Result SharedValidator::OnElemSegmentElemExpr_RefNull(const Location& loc,
-                                                      Type type) {
-  return CheckType(loc, type, elems_.back().element, "elem expression");
-}
-
-Result SharedValidator::OnElemSegmentElemExpr_RefFunc(const Location& loc,
-                                                      Var func_var) {
-  Result result = Result::Ok;
-  result |=
-      CheckType(loc, Type::FuncRef, elems_.back().element, "elem expression");
-  result |= CheckFuncIndex(func_var);
-  declared_funcs_.insert(func_var.index());
-  return result;
-}
-
-Result SharedValidator::OnElemSegmentElemExpr_Other(const Location& loc) {
-  return PrintError(loc,
-                    "invalid elem expression expression; must be either "
-                    "ref.null or ref.func.");
-}
-
 void SharedValidator::OnDataCount(Index count) {
   data_segments_ = count;
 }
@@ -518,6 +509,17 @@ Result SharedValidator::CheckAlign(const Location& loc,
   return Result::Ok;
 }
 
+Result SharedValidator::CheckOffset(const Location& loc,
+                                    Address offset,
+                                    const Limits& limits) {
+  if ((!limits.is_64) && (offset > UINT32_MAX)) {
+    PrintError(loc, "offset must be less than or equal to 0xffffffff");
+    return Result::Error;
+  }
+
+  return Result::Ok;
+}
+
 Result SharedValidator::CheckAtomicAlign(const Location& loc,
                                          Address alignment,
                                          Address natural_alignment) {
@@ -579,11 +581,13 @@ Result SharedValidator::OnAtomicFence(const Location& loc,
 Result SharedValidator::OnAtomicLoad(const Location& loc,
                                      Opcode opcode,
                                      Var memidx,
-                                     Address alignment) {
+                                     Address alignment,
+                                     Address offset) {
   Result result = CheckInstr(opcode, loc);
   MemoryType mt;
   result |= CheckMemoryIndex(memidx, &mt);
   result |= CheckAtomicAlign(loc, alignment, opcode.GetMemorySize());
+  result |= CheckOffset(loc, offset, mt.limits);
   result |= typechecker_.OnAtomicLoad(opcode, mt.limits);
   return result;
 }
@@ -591,11 +595,13 @@ Result SharedValidator::OnAtomicLoad(const Location& loc,
 Result SharedValidator::OnAtomicNotify(const Location& loc,
                                        Opcode opcode,
                                        Var memidx,
-                                       Address alignment) {
+                                       Address alignment,
+                                       Address offset) {
   Result result = CheckInstr(opcode, loc);
   MemoryType mt;
   result |= CheckMemoryIndex(memidx, &mt);
   result |= CheckAtomicAlign(loc, alignment, opcode.GetMemorySize());
+  result |= CheckOffset(loc, offset, mt.limits);
   result |= typechecker_.OnAtomicNotify(opcode, mt.limits);
   return result;
 }
@@ -603,11 +609,13 @@ Result SharedValidator::OnAtomicNotify(const Location& loc,
 Result SharedValidator::OnAtomicRmwCmpxchg(const Location& loc,
                                            Opcode opcode,
                                            Var memidx,
-                                           Address alignment) {
+                                           Address alignment,
+                                           Address offset) {
   Result result = CheckInstr(opcode, loc);
   MemoryType mt;
   result |= CheckMemoryIndex(memidx, &mt);
   result |= CheckAtomicAlign(loc, alignment, opcode.GetMemorySize());
+  result |= CheckOffset(loc, offset, mt.limits);
   result |= typechecker_.OnAtomicRmwCmpxchg(opcode, mt.limits);
   return result;
 }
@@ -615,11 +623,13 @@ Result SharedValidator::OnAtomicRmwCmpxchg(const Location& loc,
 Result SharedValidator::OnAtomicRmw(const Location& loc,
                                     Opcode opcode,
                                     Var memidx,
-                                    Address alignment) {
+                                    Address alignment,
+                                    Address offset) {
   Result result = CheckInstr(opcode, loc);
   MemoryType mt;
   result |= CheckMemoryIndex(memidx, &mt);
   result |= CheckAtomicAlign(loc, alignment, opcode.GetMemorySize());
+  result |= CheckOffset(loc, offset, mt.limits);
   result |= typechecker_.OnAtomicRmw(opcode, mt.limits);
   return result;
 }
@@ -627,11 +637,13 @@ Result SharedValidator::OnAtomicRmw(const Location& loc,
 Result SharedValidator::OnAtomicStore(const Location& loc,
                                       Opcode opcode,
                                       Var memidx,
-                                      Address alignment) {
+                                      Address alignment,
+                                      Address offset) {
   Result result = CheckInstr(opcode, loc);
   MemoryType mt;
   result |= CheckMemoryIndex(memidx, &mt);
   result |= CheckAtomicAlign(loc, alignment, opcode.GetMemorySize());
+  result |= CheckOffset(loc, offset, mt.limits);
   result |= typechecker_.OnAtomicStore(opcode, mt.limits);
   return result;
 }
@@ -639,11 +651,13 @@ Result SharedValidator::OnAtomicStore(const Location& loc,
 Result SharedValidator::OnAtomicWait(const Location& loc,
                                      Opcode opcode,
                                      Var memidx,
-                                     Address alignment) {
+                                     Address alignment,
+                                     Address offset) {
   Result result = CheckInstr(opcode, loc);
   MemoryType mt;
   result |= CheckMemoryIndex(memidx, &mt);
   result |= CheckAtomicAlign(loc, alignment, opcode.GetMemorySize());
+  result |= CheckOffset(loc, offset, mt.limits);
   result |= typechecker_.OnAtomicWait(opcode, mt.limits);
   return result;
 }
@@ -707,9 +721,16 @@ Result SharedValidator::OnCallIndirect(const Location& loc,
                                        Var table_var) {
   Result result = CheckInstr(Opcode::CallIndirect, loc);
   FuncType func_type;
+  TableType table_type;
   result |= CheckFuncTypeIndex(sig_var, &func_type);
-  result |= CheckTableIndex(table_var);
+  result |= CheckTableIndex(table_var, &table_type);
-  result |= typechecker_.OnCallIndirect(func_type.params, func_type.results);
+  if (table_type.element != Type::FuncRef) {
+    result |= PrintError(
+        loc,
+        "type mismatch: call_indirect must reference table of funcref type");
+  }
+  result |= typechecker_.OnCallIndirect(func_type.params, func_type.results,
+                                        table_type.limits);
   return result;
 }
 
@@ -850,11 +871,13 @@ Result SharedValidator::OnIf(const Location& loc, Type sig_type) {
 Result SharedValidator::OnLoad(const Location& loc,
                                Opcode opcode,
                                Var memidx,
-                               Address alignment) {
+                               Address alignment,
+                               Address offset) {
   Result result = CheckInstr(opcode, loc);
   MemoryType mt;
   result |= CheckMemoryIndex(memidx, &mt);
   result |= CheckAlign(loc, alignment, opcode.GetMemorySize());
+  result |= CheckOffset(loc, offset, mt.limits);
   result |= typechecker_.OnLoad(opcode, mt.limits);
   return result;
 }
@@ -862,11 +885,13 @@ Result SharedValidator::OnLoad(const Location& loc,
 Result SharedValidator::OnLoadSplat(const Location& loc,
                                     Opcode opcode,
                                     Var memidx,
-                                    Address alignment) {
+                                    Address alignment,
+                                    Address offset) {
   Result result = CheckInstr(opcode, loc);
   MemoryType mt;
   result |= CheckMemoryIndex(memidx, &mt);
   result |= CheckAlign(loc, alignment, opcode.GetMemorySize());
+  result |= CheckOffset(loc, offset, mt.limits);
   result |= typechecker_.OnLoad(opcode, mt.limits);
   return result;
 }
@@ -874,11 +899,13 @@ Result SharedValidator::OnLoadSplat(const Location& loc,
 Result SharedValidator::OnLoadZero(const Location& loc,
                                    Opcode opcode,
                                    Var memidx,
-                                   Address alignment) {
+                                   Address alignment,
+                                   Address offset) {
   Result result = CheckInstr(opcode, loc);
   MemoryType mt;
   result |= CheckMemoryIndex(memidx, &mt);
   result |= CheckAlign(loc, alignment, opcode.GetMemorySize());
+  result |= CheckOffset(loc, offset, mt.limits);
   result |= typechecker_.OnLoad(opcode, mt.limits);
   return result;
 }
@@ -920,14 +947,14 @@ Result SharedValidator::OnLoop(const Location& loc, Type sig_type) {
 }
 
 Result SharedValidator::OnMemoryCopy(const Location& loc,
-                                     Var srcmemidx,
+                                     Var destmemidx,
-                                     Var destmemidx) {
+                                     Var srcmemidx) {
   Result result = CheckInstr(Opcode::MemoryCopy, loc);
   MemoryType srcmt;
   MemoryType dstmt;
-  result |= CheckMemoryIndex(srcmemidx, &srcmt);
   result |= CheckMemoryIndex(destmemidx, &dstmt);
-  result |= typechecker_.OnMemoryCopy(srcmt.limits, dstmt.limits);
+  result |= CheckMemoryIndex(srcmemidx, &srcmt);
+  result |= typechecker_.OnMemoryCopy(dstmt.limits, srcmt.limits);
   return result;
 }
 
@@ -983,7 +1010,7 @@ Result SharedValidator::OnRefFunc(const Location& loc, Var func_var) {
       check_declared_funcs_.push_back(func_var);
     }
     Index func_type = GetFunctionTypeIndex(func_var.index());
-    result |= typechecker_.OnRefFuncExpr(func_type);
+    result |= typechecker_.OnRefFuncExpr(func_type, in_init_expr_);
   }
   return result;
 }
@@ -1018,9 +1045,15 @@ Result SharedValidator::OnReturnCallIndirect(const Location& loc,
                                              Var sig_var,
                                              Var table_var) {
   Result result = CheckInstr(Opcode::CallIndirect, loc);
-  result |= CheckTableIndex(table_var);
   FuncType func_type;
+  TableType table_type;
   result |= CheckFuncTypeIndex(sig_var, &func_type);
+  result |= CheckTableIndex(table_var, &table_type);
+  if (table_type.element != Type::FuncRef) {
+    result |= PrintError(loc,
+                         "type mismatch: return_call_indirect must reference "
+                         "table of funcref type");
+  }
   result |=
       typechecker_.OnReturnCallIndirect(func_type.params, func_type.results);
   return result;
@@ -1058,11 +1091,13 @@ Result SharedValidator::OnSimdLoadLane(const Location& loc,
                                        Opcode opcode,
                                        Var memidx,
                                        Address alignment,
+                                       Address offset,
                                        uint64_t value) {
   Result result = CheckInstr(opcode, loc);
   MemoryType mt;
   result |= CheckMemoryIndex(memidx, &mt);
   result |= CheckAlign(loc, alignment, opcode.GetMemorySize());
+  result |= CheckOffset(loc, offset, mt.limits);
   result |= typechecker_.OnSimdLoadLane(opcode, mt.limits, value);
   return result;
 }
@@ -1071,11 +1106,13 @@ Result SharedValidator::OnSimdStoreLane(const Location& loc,
                                         Opcode opcode,
                                         Var memidx,
                                         Address alignment,
+                                        Address offset,
                                         uint64_t value) {
   Result result = CheckInstr(opcode, loc);
   MemoryType mt;
   result |= CheckMemoryIndex(memidx, &mt);
   result |= CheckAlign(loc, alignment, opcode.GetMemorySize());
+  result |= CheckOffset(loc, offset, mt.limits);
   result |= typechecker_.OnSimdStoreLane(opcode, mt.limits, value);
   return result;
 }
@@ -1091,11 +1128,13 @@ Result SharedValidator::OnSimdShuffleOp(const Location& loc,
 Result SharedValidator::OnStore(const Location& loc,
                                 Opcode opcode,
                                 Var memidx,
-                                Address alignment) {
+                                Address alignment,
+                                Address offset) {
   Result result = CheckInstr(opcode, loc);
   MemoryType mt;
   result |= CheckMemoryIndex(memidx, &mt);
   result |= CheckAlign(loc, alignment, opcode.GetMemorySize());
+  result |= CheckOffset(loc, offset, mt.limits);
   result |= typechecker_.OnStore(opcode, mt.limits);
   return result;
 }
@@ -1108,7 +1147,7 @@ Result SharedValidator::OnTableCopy(const Location& loc,
   TableType src_table;
   result |= CheckTableIndex(dst_var, &dst_table);
   result |= CheckTableIndex(src_var, &src_table);
-  result |= typechecker_.OnTableCopy();
+  result |= typechecker_.OnTableCopy(dst_table.limits, src_table.limits);
   result |= CheckType(loc, src_table.element, dst_table.element, "table.copy");
   return result;
 }
@@ -1117,7 +1156,7 @@ Result SharedValidator::OnTableFill(const Location& loc, Var table_var) {
   Result result = CheckInstr(Opcode::TableFill, loc);
   TableType table_type;
   result |= CheckTableIndex(table_var, &table_type);
-  result |= typechecker_.OnTableFill(table_type.element);
+  result |= typechecker_.OnTableFill(table_type.element, table_type.limits);
   return result;
 }
 
@@ -1125,7 +1164,7 @@ Result SharedValidator::OnTableGet(const Location& loc, Var table_var) {
   Result result = CheckInstr(Opcode::TableGet, loc);
   TableType table_type;
   result |= CheckTableIndex(table_var, &table_type);
-  result |= typechecker_.OnTableGet(table_type.element);
+  result |= typechecker_.OnTableGet(table_type.element, table_type.limits);
   return result;
 }
 
@@ -1133,7 +1172,7 @@ Result SharedValidator::OnTableGrow(const Location& loc, Var table_var) {
   Result result = CheckInstr(Opcode::TableGrow, loc);
   TableType table_type;
   result |= CheckTableIndex(table_var, &table_type);
-  result |= typechecker_.OnTableGrow(table_type.element);
+  result |= typechecker_.OnTableGrow(table_type.element, table_type.limits);
   return result;
 }
 
@@ -1145,7 +1184,7 @@ Result SharedValidator::OnTableInit(const Location& loc,
   ElemType elem_type;
   result |= CheckTableIndex(table_var, &table_type);
   result |= CheckElemSegmentIndex(segment_var, &elem_type);
-  result |= typechecker_.OnTableInit(table_var.index(), segment_var.index());
+  result |= typechecker_.OnTableInit(segment_var.index(), table_type.limits);
   result |= CheckType(loc, elem_type.element, table_type.element, "table.init");
   return result;
 }
@@ -1154,14 +1193,15 @@ Result SharedValidator::OnTableSet(const Location& loc, Var table_var) {
   Result result = CheckInstr(Opcode::TableSet, loc);
   TableType table_type;
   result |= CheckTableIndex(table_var, &table_type);
-  result |= typechecker_.OnTableSet(table_type.element);
+  result |= typechecker_.OnTableSet(table_type.element, table_type.limits);
   return result;
 }
 
 Result SharedValidator::OnTableSize(const Location& loc, Var table_var) {
   Result result = CheckInstr(Opcode::TableSize, loc);
-  result |= CheckTableIndex(table_var);
+  TableType tt;
-  result |= typechecker_.OnTableSize();
+  result |= CheckTableIndex(table_var, &tt);
+  result |= typechecker_.OnTableSize(tt.limits);
   return result;
 }
 

third_party/wasm2c/src/test-circular-array.cc

@@ -1,284 +0,0 @@
-/*
- * Copyright 2017 WebAssembly Community Group participants
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "gtest/gtest.h"
-
-#include <memory>
-
-#include "wabt/circular-array.h"
-
-using namespace wabt;
-
-namespace {
-
-struct TestObject {
-  static int construct_count;
-  static int destruct_count;
-
-  TestObject(int data = 0, int data2 = 0) : data(data), data2(data2) {
-    construct_count++;
-  }
-
-  TestObject(const TestObject& other) {
-    *this = other;
-    construct_count++;
-  }
-
-  TestObject& operator=(const TestObject& other) {
-    data = other.data;
-    data2 = other.data2;
-    return *this;
-  }
-
-  TestObject(TestObject&& other) { *this = std::move(other); }
-
-  TestObject& operator=(TestObject&& other) {
-    data = other.data;
-    data2 = other.data2;
-    other.moved = true;
-    return *this;
-  }
-
-  ~TestObject() {
-    if (!moved) {
-      destruct_count++;
-    }
-  }
-
-  int data = 0;
-  int data2 = 0;
-  bool moved = false;
-};
-
-// static
-int TestObject::construct_count = 0;
-// static
-int TestObject::destruct_count = 0;
-
-class CircularArrayTest : public ::testing::Test {
- protected:
-  virtual void SetUp() {
-    // Reset to 0 even if the previous test leaked objects to keep the tests
-    // independent.
-    TestObject::construct_count = 0;
-    TestObject::destruct_count = 0;
-  }
-
-  virtual void TearDown() {
-    ASSERT_EQ(0, TestObject::construct_count - TestObject::destruct_count)
-        << "construct count: " << TestObject::construct_count
-        << ", destruct_count: " << TestObject::destruct_count;
-  }
-
-  template <size_t N>
-  void AssertCircularArrayEq(const CircularArray<TestObject, N>& ca,
-                             const std::vector<int>& expected) {
-    if (expected.empty()) {
-      ASSERT_EQ(0U, ca.size());
-      ASSERT_TRUE(ca.empty());
-    } else {
-      ASSERT_EQ(expected.size(), ca.size());
-      ASSERT_FALSE(ca.empty());
-
-      ASSERT_EQ(expected.front(), ca.front().data);
-      ASSERT_EQ(expected.back(), ca.back().data);
-
-      for (size_t i = 0; i < ca.size(); ++i) {
-        ASSERT_EQ(expected[i], ca.at(i).data);
-        ASSERT_EQ(expected[i], ca[i].data);
-      }
-    }
-  }
-};
-
-}  // end anonymous namespace
-
-// Basic API tests
-
-TEST_F(CircularArrayTest, default_constructor) {
-  CircularArray<TestObject, 2> ca;
-}
-
-TEST_F(CircularArrayTest, at) {
-  CircularArray<TestObject, 2> ca;
-  ca.push_back(TestObject(1));
-  ca.push_back(TestObject(2));
-
-  ASSERT_EQ(1, ca.at(0).data);
-  ASSERT_EQ(2, ca.at(1).data);
-}
-
-TEST_F(CircularArrayTest, const_at) {
-  CircularArray<TestObject, 2> ca;
-  const auto& cca = ca;
-
-  ca.push_back(TestObject(1));
-  ca.push_back(TestObject(2));
-
-  ASSERT_EQ(1, cca.at(0).data);
-  ASSERT_EQ(2, cca.at(1).data);
-}
-
-TEST_F(CircularArrayTest, operator_brackets) {
-  CircularArray<TestObject, 2> ca;
-  ca.push_back(TestObject(1));
-  ca.push_back(TestObject(2));
-
-  ASSERT_EQ(1, ca[0].data);
-  ASSERT_EQ(2, ca[1].data);
-}
-
-TEST_F(CircularArrayTest, const_operator_brackets) {
-  CircularArray<TestObject, 2> ca;
-  const auto& cca = ca;
-
-  ca.push_back(TestObject(1));
-  ca.push_back(TestObject(2));
-
-  ASSERT_EQ(1, cca[0].data);
-  ASSERT_EQ(2, cca[1].data);
-}
-
-TEST_F(CircularArrayTest, back) {
-  CircularArray<TestObject, 2> ca;
-
-  ca.push_back(TestObject(1));
-  ASSERT_EQ(1, ca.back().data);
-
-  ca.push_back(TestObject(2));
-  ASSERT_EQ(2, ca.back().data);
-}
-
-TEST_F(CircularArrayTest, const_back) {
-  CircularArray<TestObject, 2> ca;
-  const auto& cca = ca;
-
-  ca.push_back(TestObject(1));
-  ASSERT_EQ(1, cca.back().data);
-
-  ca.push_back(TestObject(2));
-  ASSERT_EQ(2, cca.back().data);
-}
-
-TEST_F(CircularArrayTest, empty) {
-  CircularArray<TestObject, 2> ca;
-
-  ASSERT_TRUE(ca.empty());
-
-  ca.push_back(TestObject(1));
-  ASSERT_FALSE(ca.empty());
-
-  ca.push_back(TestObject(2));
-  ASSERT_FALSE(ca.empty());
-
-  ca.pop_back();
-  ASSERT_FALSE(ca.empty());
-
-  ca.pop_back();
-  ASSERT_TRUE(ca.empty());
-}
-
-TEST_F(CircularArrayTest, front) {
-  CircularArray<TestObject, 2> ca;
-
-  ca.push_back(TestObject(1));
-  ASSERT_EQ(1, ca.front().data);
-
-  ca.push_back(TestObject(2));
-  ASSERT_EQ(1, ca.front().data);
-}
-
-TEST_F(CircularArrayTest, const_front) {
-  CircularArray<TestObject, 2> ca;
-  const auto& cca = ca;
-
-  ca.push_back(TestObject(1));
-  ASSERT_EQ(1, cca.front().data);
-
-  ca.push_back(TestObject(2));
-  ASSERT_EQ(1, cca.front().data);
-}
-
-TEST_F(CircularArrayTest, size) {
-  CircularArray<TestObject, 2> ca;
-
-  ASSERT_EQ(0U, ca.size());
-
-  ca.push_back(TestObject(1));
-  ASSERT_EQ(1U, ca.size());
-
-  ca.push_back(TestObject(2));
-  ASSERT_EQ(2U, ca.size());
-
-  ca.pop_back();
-  ASSERT_EQ(1U, ca.size());
-
-  ca.pop_back();
-  ASSERT_EQ(0U, ca.size());
-}
-
-TEST_F(CircularArrayTest, clear) {
-  CircularArray<TestObject, 2> ca;
-
-  ca.push_back(TestObject(1));
-  ca.push_back(TestObject(2));
-  ASSERT_EQ(2U, ca.size());
-
-  ca.clear();
-  ASSERT_EQ(0U, ca.size());
-}
-
-// More involved tests
-
-TEST_F(CircularArrayTest, circular) {
-  CircularArray<TestObject, 4> ca;
-
-  ca.push_back(TestObject(1));
-  AssertCircularArrayEq(ca, {1});
-
-  ca.push_back(TestObject(2));
-  AssertCircularArrayEq(ca, {1, 2});
-
-  ca.push_back(TestObject(3));
-  AssertCircularArrayEq(ca, {1, 2, 3});
-
-  ca.pop_front();
-  AssertCircularArrayEq(ca, {2, 3});
-
-  ca.push_back(TestObject(4));
-  AssertCircularArrayEq(ca, {2, 3, 4});
-
-  ca.pop_front();
-  AssertCircularArrayEq(ca, {3, 4});
-
-  ca.pop_front();
-  AssertCircularArrayEq(ca, {4});
-
-  ca.push_back(TestObject(5));
-  AssertCircularArrayEq(ca, {4, 5});
-
-  ca.push_back(TestObject(6));
-  AssertCircularArrayEq(ca, {4, 5, 6});
-
-  ca.pop_back();
-  AssertCircularArrayEq(ca, {4, 5});
-
-  ca.pop_back();
-  AssertCircularArrayEq(ca, {4});
-
-  ca.pop_front();
-  AssertCircularArrayEq(ca, {});
-}

third_party/wasm2c/src/test-interp.cc

@@ -491,7 +491,8 @@ TEST_F(InterpTest, Rot13) {
 
   std::string string_data = "Hello, WebAssembly!";
 
-  auto memory = Memory::New(store_, MemoryType{Limits{1}});
+  auto memory =
+      Memory::New(store_, MemoryType{Limits{1}, WABT_DEFAULT_PAGE_SIZE});
 
   auto fill_buf = [&](Thread& thread, const Values& params, Values& results,
                       Trap::Ptr* out_trap) -> Result {
@@ -505,8 +506,13 @@ TEST_F(InterpTest, Rot13) {
 
     EXPECT_LT(ptr + size, memory->ByteSize());
 
+#if WABT_BIG_ENDIAN
+    std::copy(string_data.rbegin(), string_data.rbegin() + size,
+              memory->UnsafeData() + memory->ByteSize() - ptr - size);
+#else
     std::copy(string_data.begin(), string_data.begin() + size,
               memory->UnsafeData() + ptr);
+#endif
 
     results[0].Set(size);
     return Result::Ok;
@@ -527,8 +533,14 @@ TEST_F(InterpTest, Rot13) {
     EXPECT_LT(ptr + size, memory->ByteSize());
 
     string_data.resize(size);
+#if WABT_BIG_ENDIAN
+    std::copy(memory->UnsafeData() + memory->ByteSize() - ptr - size,
+              memory->UnsafeData() + memory->ByteSize() - ptr,
+              string_data.rbegin());
+#else
     std::copy(memory->UnsafeData() + ptr, memory->UnsafeData() + ptr + size,
               string_data.begin());
+#endif
 
     return Result::Ok;
   };
@@ -648,7 +660,7 @@ TEST_F(InterpGCTest, Collect_InstanceImport) {
                          [](Thread& thread, const Values&, Values&,
                             Trap::Ptr*) -> Result { return Result::Ok; });
   auto t = Table::New(store_, TableType{ValueType::FuncRef, Limits{0}});
-  auto m = Memory::New(store_, MemoryType{Limits{0}});
+  auto m = Memory::New(store_, MemoryType{Limits{0}, WABT_DEFAULT_PAGE_SIZE});
   auto g = Global::New(store_, GlobalType{ValueType::I32, Mutability::Const},
                        Value::Make(5));
 

third_party/wasm2c/src/tools/wasm2c.cc

@@ -41,7 +41,6 @@ static int s_verbose;
 static std::string s_infile;
 static std::string s_outfile;
 static unsigned int s_num_outputs = 1;
-static Features s_features;
 static WriteCOptions s_write_c_options;
 static bool s_read_debug_names = true;
 static std::unique_ptr<FileStream> s_log_stream;
@@ -60,7 +59,8 @@ examples:
 
 static const std::string supported_features[] = {
     "multi-memory", "multi-value", "sign-extension", "saturating-float-to-int",
-    "exceptions",   "memory64",    "extended-const", "simd"};
+    "exceptions",   "memory64",    "extended-const", "simd",
+    "threads",      "tail-call"};
 
 static bool IsFeatureSupported(const std::string& feature) {
   return std::find(std::begin(supported_features), std::end(supported_features),
@@ -91,7 +91,7 @@ static void ParseOptions(int argc, char** argv) {
       "names section is used. If that is not present the name of the input\n"
       "file is used as the default.\n",
       [](const char* argument) { s_write_c_options.module_name = argument; });
-  s_features.AddOptions(&parser);
+  s_write_c_options.features.AddOptions(&parser);
   parser.AddOption("no-debug-names", "Ignore debug names in the binary file",
                    []() { s_read_debug_names = false; });
   parser.AddArgument("filename", OptionParser::ArgumentCount::One,
@@ -102,10 +102,11 @@ static void ParseOptions(int argc, char** argv) {
   parser.Parse(argc, argv);
 
   bool any_non_supported_feature = false;
-#define WABT_FEATURE(variable, flag, default_, help)   \
+#define WABT_FEATURE(variable, flag, default_, help)                   \
-  any_non_supported_feature |=                         \
+  any_non_supported_feature |=                                         \
-      (s_features.variable##_enabled() != default_) && \
+      (s_write_c_options.features.variable##_enabled() != default_) && \
-      s_features.variable##_enabled() && !IsFeatureSupported(flag);
+      s_write_c_options.features.variable##_enabled() &&               \
+      !IsFeatureSupported(flag);
 #include "wabt/feature.def"
 #undef WABT_FEATURE
 
@@ -116,16 +117,6 @@ static void ParseOptions(int argc, char** argv) {
   }
 }
 
-// TODO(binji): copied from binary-writer-spec.cc, probably should share.
-static std::string_view strip_extension(std::string_view s) {
-  std::string_view ext = s.substr(s.find_last_of('.'));
-  std::string_view result = s;
-
-  if (ext == ".c")
-    result.remove_suffix(ext.length());
-  return result;
-}
-
 Result Wasm2cMain(Errors& errors) {
   if (s_num_outputs < 1) {
     fprintf(stderr, "Number of output files must be positive.\n");
@@ -138,11 +129,12 @@ Result Wasm2cMain(Errors& errors) {
   Module module;
   const bool kStopOnFirstError = true;
   const bool kFailOnCustomSectionError = true;
-  ReadBinaryOptions options(s_features, s_log_stream.get(), s_read_debug_names,
+  ReadBinaryOptions options(s_write_c_options.features, s_log_stream.get(),
-                            kStopOnFirstError, kFailOnCustomSectionError);
+                            s_read_debug_names, kStopOnFirstError,
+                            kFailOnCustomSectionError);
   CHECK_RESULT(ReadBinaryIr(s_infile.c_str(), file_data.data(),
                             file_data.size(), options, &errors, &module));
-  CHECK_RESULT(ValidateModule(&module, &errors, s_features));
+  CHECK_RESULT(ValidateModule(&module, &errors, s_write_c_options.features));
   CHECK_RESULT(GenerateNames(&module));
   /* TODO(binji): This shouldn't fail; if a name can't be applied
    * (because the index is invalid, say) it should just be skipped. */
@@ -150,12 +142,12 @@ Result Wasm2cMain(Errors& errors) {
 
   if (!s_outfile.empty()) {
     std::string header_name_full =
-        std::string(strip_extension(s_outfile)) + ".h";
+        std::string(wabt::StripExtension(s_outfile)) + ".h";
     std::vector<FileStream> c_streams;
     if (s_num_outputs == 1) {
       c_streams.emplace_back(s_outfile.c_str());
     } else {
-      std::string output_prefix{strip_extension(s_outfile)};
+      std::string output_prefix{wabt::StripExtension(s_outfile)};
       for (unsigned int i = 0; i < s_num_outputs; i++) {
         c_streams.emplace_back(output_prefix + "_" + std::to_string(i) + ".c");
       }
@@ -179,7 +171,7 @@ Result Wasm2cMain(Errors& errors) {
                           s_write_c_options));
     } else {
       std::string header_impl_name_full =
-          std::string(strip_extension(s_outfile)) + "-impl.h";
+          std::string(wabt::StripExtension(s_outfile)) + "-impl.h";
       FileStream h_impl_stream(header_impl_name_full);
       std::string_view header_impl_name = GetBasename(header_impl_name_full);
       CHECK_RESULT(WriteC(std::move(c_stream_ptrs), &h_stream, &h_impl_stream,

third_party/wasm2c/src/type-checker.cc

@@ -30,7 +30,14 @@ std::string TypesToString(const TypeVector& types,
   }
 
   for (size_t i = 0; i < types.size(); ++i) {
-    result += types[i].GetName();
+    Type ty = types[i];
+    // NOTE: Reference (and GetName) is also used by (e.g.) objdump, which does
+    // not apply validation. do this here so as to not break that.
+    if (ty == Type::Reference && ty.GetReferenceIndex() == kInvalidIndex) {
+      result += "reference";
+    } else {
+      result += types[i].GetName();
+    }
     if (i < types.size() - 1) {
       result += ", ";
     }
@@ -254,7 +261,7 @@ Result TypeChecker::CheckSignature(const TypeVector& sig, const char* desc) {
   for (size_t i = 0; i < sig.size(); ++i) {
     result |= PeekAndCheckType(sig.size() - i - 1, sig[i]);
   }
-  PrintStackIfFailed(result, desc, sig);
+  PrintStackIfFailedV(result, desc, sig, /*is_end=*/false);
   return result;
 }
 
@@ -505,29 +512,24 @@ Result TypeChecker::OnCall(const TypeVector& param_types,
 }
 
 Result TypeChecker::OnCallIndirect(const TypeVector& param_types,
-                                   const TypeVector& result_types) {
+                                   const TypeVector& result_types,
-  Result result = PopAndCheck1Type(Type::I32, "call_indirect");
+                                   const Limits& table_limits) {
+  Result result = PopAndCheck1Type(table_limits.is_64 ? Type::I64 : Type::I32,
+                                   "call_indirect");
   result |= PopAndCheckCall(param_types, result_types, "call_indirect");
   return result;
 }
 
 Result TypeChecker::OnIndexedFuncRef(Index* out_index) {
   Type type;
-  CHECK_RESULT(PeekType(0, &type));
+  Result result = PeekType(0, &type);
-  Result result = Result::Ok;
+  if (!type.IsReferenceWithIndex()) {
-  if (!(type == Type::Any || type.IsReferenceWithIndex())) {
+    type = Type::Reference;
-    TypeVector actual;
-    actual.push_back(type);
-    std::string message =
-        "type mismatch in call_ref, expected reference but got " +
-        TypesToString(actual);
-    PrintError("%s", message.c_str());
-    result = Result::Error;
   }
+  result |= PopAndCheck1Type(type, "call_ref");
   if (Succeeded(result)) {
     *out_index = type.GetReferenceIndex();
   }
-  result |= DropTypes(1);
   return result;
 }
 
@@ -705,15 +707,15 @@ Result TypeChecker::OnLoop(const TypeVector& param_types,
   return result;
 }
 
-Result TypeChecker::OnMemoryCopy(const Limits& src_limits,
+Result TypeChecker::OnMemoryCopy(const Limits& dst_limits,
-                                 const Limits& dst_limits) {
+                                 const Limits& src_limits) {
   Limits size_limits = src_limits;
   // The memory64 proposal specifies that the type of the size argument should
   // be the mimimum of the two memory types.
   if (src_limits.is_64 && !dst_limits.is_64) {
     size_limits = dst_limits;
   }
-  return CheckOpcode3(Opcode::MemoryCopy, &src_limits, &dst_limits,
+  return CheckOpcode3(Opcode::MemoryCopy, &dst_limits, &src_limits,
                       &size_limits);
 }
 
@@ -740,45 +742,61 @@ Result TypeChecker::OnMemorySize(const Limits& limits) {
   return Result::Ok;
 }
 
-Result TypeChecker::OnTableCopy() {
+Result TypeChecker::OnTableCopy(const Limits& dst_limits,
-  return CheckOpcode3(Opcode::TableCopy);
+                                const Limits& src_limits) {
+  Limits size_limits = src_limits;
+  // The memory64 proposal specifies that the type of the size argument should
+  // be the mimimum of the two table types.
+  if (src_limits.is_64 && !dst_limits.is_64) {
+    size_limits = dst_limits;
+  }
+  return CheckOpcode3(Opcode::TableCopy, &dst_limits, &src_limits,
+                      &size_limits);
 }
 
 Result TypeChecker::OnElemDrop(uint32_t segment) {
   return Result::Ok;
 }
 
-Result TypeChecker::OnTableInit(uint32_t table, uint32_t segment) {
+Result TypeChecker::OnTableInit(uint32_t segment, const Limits& limits) {
-  return CheckOpcode3(Opcode::TableInit);
+  return CheckOpcode3(Opcode::TableInit, &limits);
 }
 
-Result TypeChecker::OnTableGet(Type elem_type) {
+Result TypeChecker::OnTableGet(Type elem_type, const Limits& limits) {
-  Result result = PopAndCheck1Type(Type::I32, "table.get");
+  Result result = CheckOpcode1(Opcode::TableGet, &limits);
   PushType(elem_type);
   return result;
 }
 
-Result TypeChecker::OnTableSet(Type elem_type) {
+Result TypeChecker::OnTableSet(Type elem_type, const Limits& limits) {
-  return PopAndCheck2Types(Type::I32, elem_type, "table.set");
+  return PopAndCheck2Types(limits.IndexType(), elem_type, "table.set");
 }
 
-Result TypeChecker::OnTableGrow(Type elem_type) {
+Result TypeChecker::OnTableGrow(Type elem_type, const Limits& limits) {
-  Result result = PopAndCheck2Types(elem_type, Type::I32, "table.grow");
+  Result result =
-  PushType(Type::I32);
+      PopAndCheck2Types(elem_type, limits.IndexType(), "table.grow");
+  PushType(limits.IndexType());
   return result;
 }
 
-Result TypeChecker::OnTableSize() {
+Result TypeChecker::OnTableSize(const Limits& limits) {
-  PushType(Type::I32);
+  PushType(limits.IndexType());
   return Result::Ok;
 }
 
-Result TypeChecker::OnTableFill(Type elem_type) {
+Result TypeChecker::OnTableFill(Type elem_type, const Limits& limits) {
-  return PopAndCheck3Types(Type::I32, elem_type, Type::I32, "table.fill");
+  return PopAndCheck3Types(limits.IndexType(), elem_type, limits.IndexType(),
+                           "table.fill");
 }
 
-Result TypeChecker::OnRefFuncExpr(Index func_type) {
+Result TypeChecker::OnRefFuncExpr(Index func_type, bool force_generic_funcref) {
-  if (features_.function_references_enabled()) {
+  /*
+   * In a const expression, treat ref.func as producing a generic funcref.
+   * This avoids having to implement funcref subtyping (for now) and matches
+   * the previous behavior where SharedValidator::OnElemSegmentElemExpr_RefFunc
+   * examined only the validity of the function index.
+   */
+  if (features_.function_references_enabled() && !force_generic_funcref) {
     PushType(Type(Type::Reference, func_type));
   } else {
     PushType(Type::FuncRef);
@@ -794,18 +812,10 @@ Result TypeChecker::OnRefNullExpr(Type type) {
 Result TypeChecker::OnRefIsNullExpr() {
   Type type;
   Result result = PeekType(0, &type);
-  if (!(type == Type::Any || type.IsRef())) {
+  if (!type.IsRef()) {
-    TypeVector actual;
+    type = Type::Reference;
-    if (Succeeded(result)) {
-      actual.push_back(type);
-    }
-    std::string message =
-        "type mismatch in ref.is_null, expected reference but got " +
-        TypesToString(actual);
-    PrintError("%s", message.c_str());
-    result = Result::Error;
   }
-  result |= DropTypes(1);
+  result |= PopAndCheck1Type(type, "ref.is_null");
   PushType(Type::I32);
   return result;
 }

third_party/wasm2c/src/validator.cc

@@ -381,8 +381,9 @@ Result Validator::EndIfExpr(IfExpr* expr) {
 }
 
 Result Validator::OnLoadExpr(LoadExpr* expr) {
-  result_ |= validator_.OnLoad(expr->loc, expr->opcode, expr->memidx,
+  result_ |=
-                               expr->opcode.GetAlignment(expr->align));
+      validator_.OnLoad(expr->loc, expr->opcode, expr->memidx,
+                        expr->opcode.GetAlignment(expr->align), expr->offset);
   return Result::Ok;
 }
 
@@ -413,7 +414,7 @@ Result Validator::EndLoopExpr(LoopExpr* expr) {
 
 Result Validator::OnMemoryCopyExpr(MemoryCopyExpr* expr) {
   result_ |=
-      validator_.OnMemoryCopy(expr->loc, expr->srcmemidx, expr->destmemidx);
+      validator_.OnMemoryCopy(expr->loc, expr->destmemidx, expr->srcmemidx);
   return Result::Ok;
 }
 
@@ -523,17 +524,13 @@ Result Validator::OnReturnCallIndirectExpr(ReturnCallIndirectExpr* expr) {
 Result Validator::OnSelectExpr(SelectExpr* expr) {
   result_ |= validator_.OnSelect(expr->loc, expr->result_type.size(),
                                  expr->result_type.data());
-  // TODO: Existing behavior fails when select fails.
-#if 0
   return Result::Ok;
-#else
-  return result_;
-#endif
 }
 
 Result Validator::OnStoreExpr(StoreExpr* expr) {
-  result_ |= validator_.OnStore(expr->loc, expr->opcode, expr->memidx,
+  result_ |=
-                                expr->opcode.GetAlignment(expr->align));
+      validator_.OnStore(expr->loc, expr->opcode, expr->memidx,
+                         expr->opcode.GetAlignment(expr->align), expr->offset);
   return Result::Ok;
 }
 
@@ -579,7 +576,8 @@ Result Validator::OnRethrowExpr(RethrowExpr* expr) {
 
 Result Validator::OnAtomicWaitExpr(AtomicWaitExpr* expr) {
   result_ |= validator_.OnAtomicWait(expr->loc, expr->opcode, expr->memidx,
-                                     expr->opcode.GetAlignment(expr->align));
+                                     expr->opcode.GetAlignment(expr->align),
+                                     expr->offset);
   return Result::Ok;
 }
 
@@ -590,32 +588,36 @@ Result Validator::OnAtomicFenceExpr(AtomicFenceExpr* expr) {
 
 Result Validator::OnAtomicNotifyExpr(AtomicNotifyExpr* expr) {
   result_ |= validator_.OnAtomicNotify(expr->loc, expr->opcode, expr->memidx,
-                                       expr->opcode.GetAlignment(expr->align));
+                                       expr->opcode.GetAlignment(expr->align),
+                                       expr->offset);
   return Result::Ok;
 }
 
 Result Validator::OnAtomicLoadExpr(AtomicLoadExpr* expr) {
   result_ |= validator_.OnAtomicLoad(expr->loc, expr->opcode, expr->memidx,
-                                     expr->opcode.GetAlignment(expr->align));
+                                     expr->opcode.GetAlignment(expr->align),
+                                     expr->offset);
   return Result::Ok;
 }
 
 Result Validator::OnAtomicStoreExpr(AtomicStoreExpr* expr) {
   result_ |= validator_.OnAtomicStore(expr->loc, expr->opcode, expr->memidx,
-                                      expr->opcode.GetAlignment(expr->align));
+                                      expr->opcode.GetAlignment(expr->align),
+                                      expr->offset);
   return Result::Ok;
 }
 
 Result Validator::OnAtomicRmwExpr(AtomicRmwExpr* expr) {
   result_ |= validator_.OnAtomicRmw(expr->loc, expr->opcode, expr->memidx,
-                                    expr->opcode.GetAlignment(expr->align));
+                                    expr->opcode.GetAlignment(expr->align),
+                                    expr->offset);
   return Result::Ok;
 }
 
 Result Validator::OnAtomicRmwCmpxchgExpr(AtomicRmwCmpxchgExpr* expr) {
-  result_ |=
+  result_ |= validator_.OnAtomicRmwCmpxchg(
-      validator_.OnAtomicRmwCmpxchg(expr->loc, expr->opcode, expr->memidx,
+      expr->loc, expr->opcode, expr->memidx,
-                                    expr->opcode.GetAlignment(expr->align));
+      expr->opcode.GetAlignment(expr->align), expr->offset);
   return Result::Ok;
 }
 
@@ -632,14 +634,14 @@ Result Validator::OnSimdLaneOpExpr(SimdLaneOpExpr* expr) {
 Result Validator::OnSimdLoadLaneExpr(SimdLoadLaneExpr* expr) {
   result_ |= validator_.OnSimdLoadLane(expr->loc, expr->opcode, expr->memidx,
                                        expr->opcode.GetAlignment(expr->align),
-                                       expr->val);
+                                       expr->offset, expr->val);
   return Result::Ok;
 }
 
 Result Validator::OnSimdStoreLaneExpr(SimdStoreLaneExpr* expr) {
   result_ |= validator_.OnSimdStoreLane(expr->loc, expr->opcode, expr->memidx,
                                         expr->opcode.GetAlignment(expr->align),
-                                        expr->val);
+                                        expr->offset, expr->val);
   return Result::Ok;
 }
 
@@ -650,13 +652,15 @@ Result Validator::OnSimdShuffleOpExpr(SimdShuffleOpExpr* expr) {
 
 Result Validator::OnLoadSplatExpr(LoadSplatExpr* expr) {
   result_ |= validator_.OnLoadSplat(expr->loc, expr->opcode, expr->memidx,
-                                    expr->opcode.GetAlignment(expr->align));
+                                    expr->opcode.GetAlignment(expr->align),
+                                    expr->offset);
   return Result::Ok;
 }
 
 Result Validator::OnLoadZeroExpr(LoadZeroExpr* expr) {
   result_ |= validator_.OnLoadZero(expr->loc, expr->opcode, expr->memidx,
-                                   expr->opcode.GetAlignment(expr->align));
+                                   expr->opcode.GetAlignment(expr->align),
+                                   expr->offset);
   return Result::Ok;
 }
 
@@ -728,7 +732,8 @@ Result Validator::CheckModule() {
 
         case ExternalKind::Memory: {
           auto&& memory = cast<MemoryImport>(f->import.get())->memory;
-          result_ |= validator_.OnMemory(field.loc, memory.page_limits);
+          result_ |= validator_.OnMemory(field.loc, memory.page_limits,
+                                         memory.page_size);
           break;
         }
 
@@ -768,7 +773,8 @@ Result Validator::CheckModule() {
   // Memory section.
   for (const ModuleField& field : module->fields) {
     if (auto* f = dyn_cast<MemoryModuleField>(&field)) {
-      result_ |= validator_.OnMemory(field.loc, f->memory.page_limits);
+      result_ |= validator_.OnMemory(field.loc, f->memory.page_limits,
+                                     f->memory.page_size);
     }
   }
 
@@ -821,7 +827,13 @@ Result Validator::CheckModule() {
 
       // Init expr.
       if (f->elem_segment.kind == SegmentKind::Active) {
-        result_ |= validator_.BeginInitExpr(field.loc, Type::I32);
+        Type offset_type = Type::I32;
+        Index table_index = module->GetTableIndex(f->elem_segment.table_var);
+        if (table_index < module->tables.size() &&
+            module->tables[table_index]->elem_limits.is_64) {
+          offset_type = Type::I64;
+        }
+        result_ |= validator_.BeginInitExpr(field.loc, offset_type);
         ExprVisitor visitor(this);
         result_ |= visitor.VisitExprList(
             const_cast<ExprList&>(f->elem_segment.offset));
@@ -830,24 +842,11 @@ Result Validator::CheckModule() {
 
       // Element expr.
       for (auto&& elem_expr : f->elem_segment.elem_exprs) {
-        if (elem_expr.size() == 1) {
+        result_ |= validator_.BeginInitExpr(elem_expr.front().loc,
-          const Expr* expr = &elem_expr.front();
+                                            f->elem_segment.elem_type);
-          switch (expr->type()) {
+        ExprVisitor visitor(this);
-            case ExprType::RefNull:
+        result_ |= visitor.VisitExprList(const_cast<ExprList&>(elem_expr));
-              result_ |= validator_.OnElemSegmentElemExpr_RefNull(
+        result_ |= validator_.EndInitExpr();
-                  expr->loc, cast<RefNullExpr>(expr)->type);
-              break;
-            case ExprType::RefFunc:
-              result_ |= validator_.OnElemSegmentElemExpr_RefFunc(
-                  expr->loc, cast<RefFuncExpr>(expr)->var);
-              break;
-            default:
-              result_ |= validator_.OnElemSegmentElemExpr_Other(expr->loc);
-              break;
-          }
-        } else if (elem_expr.size() > 1) {
-          result_ |= validator_.OnElemSegmentElemExpr_Other(field.loc);
-        }
       }
     }
   }

third_party/wasm2c/src/wabt.post.js

@@ -95,7 +95,7 @@ function allocateCString(s) {
 /// Features
 function Features(obj) {
   this.addr = Module._wabt_new_features();
-  for ([f, v] of Object.entries(FEATURES)) {
+  for (var [f, v] of Object.entries(FEATURES)) {
     this[f] = booleanOrDefault(obj[f], v);
   }
 }

third_party/wasm2c/src/wast-lexer.cc

@@ -29,7 +29,14 @@ namespace wabt {
 
 namespace {
 
+#if __clang__
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wimplicit-fallthrough"
+#endif
 #include "prebuilt/lexer-keywords.cc"
+#if __clang__
+#pragma clang diagnostic pop
+#endif
 
 }  // namespace
 
@@ -180,7 +187,7 @@ Token WastLexer::GetToken() {
 }
 
 Location WastLexer::GetLocation() {
-  auto column = [=](const char* p) {
+  auto column = [this](const char* p) {
     return std::max(1, static_cast<int>(p - line_start_ + 1));
   };
   return Location(filename_, line_, column(token_start_), column(cursor_));
@@ -278,6 +285,13 @@ bool WastLexer::ReadLineComment() {
       case kEof:
         return false;
 
+      case '\r':
+        if (PeekChar() == '\n') {
+          ReadChar();
+        }
+        Newline();
+        return true;
+
       case '\n':
         Newline();
         return true;

third_party/wasm2c/src/wast-parser.cc

@@ -593,20 +593,23 @@ Location WastParser::GetLocation() {
 }
 
 TokenType WastParser::Peek(size_t n) {
+  assert(n <= 1);
   while (tokens_.size() <= n) {
     Token cur = lexer_->GetToken();
     if (cur.token_type() != TokenType::LparAnn) {
       tokens_.push_back(cur);
     } else {
       // Custom annotation. For now, discard until matching Rpar, unless it is
-      // a code metadata annotation. In that case, we know how to parse it.
+      // a code metadata annotation or custom section. In those cases, we know
+      // how to parse it.
       if (!options_->features.annotations_enabled()) {
         Error(cur.loc, "annotations not enabled: %s", cur.to_string().c_str());
         tokens_.push_back(Token(cur.loc, TokenType::Invalid));
         continue;
       }
-      if (options_->features.code_metadata_enabled() &&
+      if ((options_->features.code_metadata_enabled() &&
-          cur.text().find("metadata.code.") == 0) {
+           cur.text().find("metadata.code.") == 0) ||
+          cur.text() == "custom") {
         tokens_.push_back(cur);
         continue;
       }
@@ -654,7 +657,8 @@ bool WastParser::PeekMatchExpr() {
 }
 
 bool WastParser::PeekMatchRefType() {
-  return options_->features.function_references_enabled() &&
+  return (options_->features.function_references_enabled() ||
+          options_->features.exceptions_enabled()) &&
          PeekMatchLpar(TokenType::Ref);
 }
 
@@ -720,6 +724,7 @@ void WastParser::ErrorUnlessOpcodeEnabled(const Token& token) {
 
 Result WastParser::ErrorExpected(const std::vector<std::string>& expected,
                                  const char* example) {
+  GetToken();
   Token token = Consume();
   std::string expected_str;
   if (!expected.empty()) {
@@ -752,7 +757,6 @@ Result WastParser::ErrorExpected(const std::vector<std::string>& expected,
 Result WastParser::ErrorIfLpar(const std::vector<std::string>& expected,
                                const char* example) {
   if (Match(TokenType::Lpar)) {
-    GetToken();
     return ErrorExpected(expected, example);
   }
   return Result::Ok;
@@ -871,7 +875,7 @@ bool WastParser::ParseElemExprOpt(ExprList* out_elem_expr) {
     }
     EXPECT(Rpar);
   } else {
-    if (ParseExpr(&exprs) != Result::Ok) {
+    if (!IsExpr(PeekPair()) || ParseExpr(&exprs) != Result::Ok) {
       return false;
     }
   }
@@ -931,6 +935,9 @@ Result WastParser::ParseValueType(Var* out_type) {
     case Type::ExternRef:
       is_enabled = options_->features.reference_types_enabled();
       break;
+    case Type::ExnRef:
+      is_enabled = options_->features.exceptions_enabled();
+      break;
     default:
       is_enabled = true;
       break;
@@ -1050,8 +1057,9 @@ bool WastParser::ParseOffsetOpt(Address* out_offset) {
       Error(token.loc, "invalid offset \"" PRIstringview "\"",
             WABT_PRINTF_STRING_VIEW_ARG(sv));
     }
-    // FIXME: make this depend on the current memory.
+    // With memory64, offsets > UINT32_MAX for i32 memories are no longer
-    if (offset64 > UINT32_MAX) {
+    // malformed (just invalid)
+    if ((!options_->features.memory64_enabled()) && (offset64 > UINT32_MAX)) {
       Error(token.loc, "offset must be less than or equal to 0xffffffff");
     }
     *out_offset = offset64;
@@ -1138,6 +1146,37 @@ Result WastParser::ParseLimits(Limits* out_limits) {
   return Result::Ok;
 }
 
+Result WastParser::ParsePageSize(uint32_t* out_page_size) {
+  WABT_TRACE(ParsePageSize);
+
+  Result result = Result::Ok;
+
+  if (PeekMatchLpar(TokenType::PageSize)) {
+    if (!options_->features.custom_page_sizes_enabled()) {
+      Error(GetLocation(), "Specifying memory page size is not allowed");
+      return Result::Error;
+    }
+    EXPECT(Lpar);
+    EXPECT(PageSize);
+    auto token = GetToken();
+    if (!token.HasLiteral()) {
+      Error(GetLocation(), "malformed custom page size");
+      return Result::Error;
+    }
+    auto sv = token.literal().text;
+    result |= ParseInt32(sv, out_page_size, ParseIntType::UnsignedOnly);
+    if (*out_page_size > UINT32_MAX || *out_page_size <= 0 ||
+        (*out_page_size & (*out_page_size - 1))) {
+      Error(GetLocation(), "malformed custom page size");
+      return Result::Error;
+    }
+    Consume();
+    EXPECT(Rpar);
+  }
+
+  return result;
+}
+
 Result WastParser::ParseNat(uint64_t* out_nat, bool is_64) {
   WABT_TRACE(ParseNat);
   if (!PeekMatch(TokenType::Nat)) {
@@ -1171,16 +1210,18 @@ Result WastParser::ParseModule(std::unique_ptr<Module>* out_module) {
       auto module_command = cast<ScriptModuleCommand>(std::move(command));
       *module = std::move(module_command->module);
     }
-  } else if (IsModuleField(PeekPair())) {
+  } else if (IsModuleField(PeekPair()) || PeekIsCustom()) {
     // Parse an inline module (i.e. one with no surrounding (module)).
     CHECK_RESULT(ParseModuleFieldList(module.get()));
+  } else if (PeekMatch(TokenType::Eof)) {
+    errors_->emplace_back(ErrorLevel::Warning, GetLocation(), "empty module");
   } else {
     ConsumeIfLpar();
     ErrorExpected({"a module field", "a module"});
   }
 
   EXPECT(Eof);
-  if (errors_->size() == 0) {
+  if (!HasError()) {
     *out_module = std::move(module);
     return Result::Ok;
   } else {
@@ -1195,7 +1236,7 @@ Result WastParser::ParseScript(std::unique_ptr<Script>* out_script) {
   // Don't consume the Lpar yet, even though it is required. This way the
   // sub-parser functions (e.g. ParseFuncModuleField) can consume it and keep
   // the parsing structure more regular.
-  if (IsModuleField(PeekPair())) {
+  if (IsModuleField(PeekPair()) || PeekIsCustom()) {
     // Parse an inline module (i.e. one with no surrounding (module)).
     auto command = std::make_unique<ModuleCommand>();
     command->module.loc = GetLocation();
@@ -1203,13 +1244,15 @@ Result WastParser::ParseScript(std::unique_ptr<Script>* out_script) {
     script->commands.emplace_back(std::move(command));
   } else if (IsCommand(PeekPair())) {
     CHECK_RESULT(ParseCommandList(script.get(), &script->commands));
+  } else if (PeekMatch(TokenType::Eof)) {
+    errors_->emplace_back(ErrorLevel::Warning, GetLocation(), "empty script");
   } else {
     ConsumeIfLpar();
     ErrorExpected({"a module field", "a command"});
   }
 
   EXPECT(Eof);
-  if (errors_->size() == 0) {
+  if (!HasError()) {
     *out_script = std::move(script);
     return Result::Ok;
   } else {
@@ -1217,9 +1260,69 @@ Result WastParser::ParseScript(std::unique_ptr<Script>* out_script) {
   }
 }
 
+Result WastParser::ParseCustomSectionAnnotation(Module* module) {
+  WABT_TRACE(ParseCustomSectionAnnotation);
+  Location loc = GetLocation();
+  Token token = Consume();
+  if (token.text() != "custom") {
+    assert(
+        !"ParseCustomSectionAnnotation should only be called if PeekIsCustom() is true");
+    return Result::Error;
+  }
+  std::string section_name;
+  CHECK_RESULT(ParseQuotedText(&section_name));
+  if (Match(TokenType::Lpar)) {
+    if (!PeekMatch(TokenType::After) && !PeekMatch(TokenType::Before)) {
+      return ErrorExpected({"before", "after"});
+    }
+    Consume();
+    switch (Peek()) {
+      case TokenType::Function:
+      case TokenType::Type:
+      case TokenType::Import:
+      case TokenType::Export:
+      case TokenType::Table:
+      case TokenType::Global:
+      case TokenType::Elem:
+      case TokenType::Data:
+      case TokenType::Memory:
+      case TokenType::Code:
+      case TokenType::Start: {
+        Consume();
+        break;
+      }
+      default: {
+        return ErrorExpected({"type", "import", "function", "table", "memory",
+                              "global", "export", "start", "elem", "code",
+                              "data"});
+      }
+    }
+    EXPECT(Rpar);
+  }
+  std::vector<uint8_t> data;
+  CHECK_RESULT(ParseTextList(&data));
+  EXPECT(Rpar);
+
+  Custom custom = Custom(loc, section_name, data);
+  module->customs.push_back(custom);
+
+  return Result::Ok;
+}
+
+bool WastParser::PeekIsCustom() {
+  // If IsLparAnn succeeds, tokens_.front() must have text, as it is an LparAnn
+  // token.
+  return options_->features.annotations_enabled() && IsLparAnn(PeekPair()) &&
+         tokens_.front().text() == "custom";
+}
+
 Result WastParser::ParseModuleFieldList(Module* module) {
   WABT_TRACE(ParseModuleFieldList);
-  while (IsModuleField(PeekPair())) {
+  while (IsModuleField(PeekPair()) || PeekIsCustom()) {
+    if (PeekIsCustom()) {
+      CHECK_RESULT(ParseCustomSectionAnnotation(module));
+      continue;
+    }
     if (Failed(ParseModuleField(module))) {
       CHECK_RESULT(Synchronize(IsModuleField));
     }
@@ -1590,6 +1693,7 @@ Result WastParser::ParseImportModuleField(Module* module) {
       Consume();
       ParseBindVarOpt(&name);
       auto import = std::make_unique<TableImport>(name);
+      CHECK_RESULT(ParseLimitsIndex(&import->table.elem_limits));
       CHECK_RESULT(ParseLimits(&import->table.elem_limits));
       CHECK_RESULT(ParseRefType(&import->table.elem_type));
       EXPECT(Rpar);
@@ -1601,8 +1705,10 @@ Result WastParser::ParseImportModuleField(Module* module) {
       Consume();
       ParseBindVarOpt(&name);
       auto import = std::make_unique<MemoryImport>(name);
+      import->memory.page_size = WABT_DEFAULT_PAGE_SIZE;
       CHECK_RESULT(ParseLimitsIndex(&import->memory.page_limits));
       CHECK_RESULT(ParseLimits(&import->memory.page_limits));
+      CHECK_RESULT(ParsePageSize(&import->memory.page_size));
       EXPECT(Rpar);
       field = std::make_unique<ImportModuleField>(std::move(import), loc);
       break;
@@ -1655,15 +1761,26 @@ Result WastParser::ParseMemoryModuleField(Module* module) {
   if (PeekMatchLpar(TokenType::Import)) {
     CheckImportOrdering(module);
     auto import = std::make_unique<MemoryImport>(name);
+    import->memory.page_size = WABT_DEFAULT_PAGE_SIZE;
     CHECK_RESULT(ParseInlineImport(import.get()));
     CHECK_RESULT(ParseLimitsIndex(&import->memory.page_limits));
     CHECK_RESULT(ParseLimits(&import->memory.page_limits));
+    CHECK_RESULT(ParsePageSize(&import->memory.page_size));
     auto field =
         std::make_unique<ImportModuleField>(std::move(import), GetLocation());
     module->AppendField(std::move(field));
   } else {
     auto field = std::make_unique<MemoryModuleField>(loc, name);
+    field->memory.page_size = WABT_DEFAULT_PAGE_SIZE;
     CHECK_RESULT(ParseLimitsIndex(&field->memory.page_limits));
+    if (PeekMatchLpar(TokenType::PageSize)) {
+      // this is the data abbreviation (no limits)
+      CHECK_RESULT(ParsePageSize(&field->memory.page_size));
+      if (!PeekMatchLpar(TokenType::Data)) {
+        ConsumeIfLpar();
+        return ErrorExpected({"inline data segment"});
+      }
+    }
     if (MatchLpar(TokenType::Data)) {
       auto data_segment_field = std::make_unique<DataSegmentModuleField>(loc);
       DataSegment& data_segment = data_segment_field->data_segment;
@@ -1674,16 +1791,17 @@ Result WastParser::ParseMemoryModuleField(Module* module) {
       ParseTextListOpt(&data_segment.data);
       EXPECT(Rpar);
 
-      uint32_t byte_size = WABT_ALIGN_UP_TO_PAGE(data_segment.data.size());
+      uint32_t num_pages = WABT_BYTES_TO_MIN_PAGES(data_segment.data.size(),
-      uint32_t page_size = WABT_BYTES_TO_PAGES(byte_size);
+                                                   field->memory.page_size);
-      field->memory.page_limits.initial = page_size;
+      field->memory.page_limits.initial = num_pages;
-      field->memory.page_limits.max = page_size;
+      field->memory.page_limits.max = num_pages;
       field->memory.page_limits.has_max = true;
 
       module->AppendField(std::move(field));
       module->AppendField(std::move(data_segment_field));
     } else {
       CHECK_RESULT(ParseLimits(&field->memory.page_limits));
+      CHECK_RESULT(ParsePageSize(&field->memory.page_size));
       module->AppendField(std::move(field));
     }
   }
@@ -1725,48 +1843,53 @@ Result WastParser::ParseTableModuleField(Module* module) {
     CheckImportOrdering(module);
     auto import = std::make_unique<TableImport>(name);
     CHECK_RESULT(ParseInlineImport(import.get()));
+    CHECK_RESULT(ParseLimitsIndex(&import->table.elem_limits));
     CHECK_RESULT(ParseLimits(&import->table.elem_limits));
     CHECK_RESULT(ParseRefType(&import->table.elem_type));
     auto field =
         std::make_unique<ImportModuleField>(std::move(import), GetLocation());
     module->AppendField(std::move(field));
-  } else if (PeekMatch(TokenType::ValueType)) {
-    Type elem_type;
-    CHECK_RESULT(ParseRefType(&elem_type));
-
-    EXPECT(Lpar);
-    EXPECT(Elem);
-
-    auto elem_segment_field = std::make_unique<ElemSegmentModuleField>(loc);
-    ElemSegment& elem_segment = elem_segment_field->elem_segment;
-    elem_segment.table_var = Var(module->tables.size(), GetLocation());
-    elem_segment.offset.push_back(std::make_unique<ConstExpr>(Const::I32(0)));
-    elem_segment.offset.back().loc = loc;
-    elem_segment.elem_type = elem_type;
-    // Syntax is either an optional list of var (legacy), or a non-empty list
-    // of elem expr.
-    ExprList elem_expr;
-    if (ParseElemExprOpt(&elem_expr)) {
-      elem_segment.elem_exprs.push_back(std::move(elem_expr));
-      // Parse the rest.
-      ParseElemExprListOpt(&elem_segment.elem_exprs);
-    } else {
-      ParseElemExprVarListOpt(&elem_segment.elem_exprs);
-    }
-    EXPECT(Rpar);
-
-    auto table_field = std::make_unique<TableModuleField>(loc, name);
-    table_field->table.elem_limits.initial = elem_segment.elem_exprs.size();
-    table_field->table.elem_limits.max = elem_segment.elem_exprs.size();
-    table_field->table.elem_limits.has_max = true;
-    table_field->table.elem_type = elem_type;
-    module->AppendField(std::move(table_field));
-    module->AppendField(std::move(elem_segment_field));
   } else {
     auto field = std::make_unique<TableModuleField>(loc, name);
-    CHECK_RESULT(ParseLimits(&field->table.elem_limits));
+    auto& table = field->table;
-    CHECK_RESULT(ParseRefType(&field->table.elem_type));
+    CHECK_RESULT(ParseLimitsIndex(&table.elem_limits));
-    module->AppendField(std::move(field));
+    if (PeekMatch(TokenType::ValueType)) {
+      Type elem_type;
+      CHECK_RESULT(ParseRefType(&elem_type));
+
+      EXPECT(Lpar);
+      EXPECT(Elem);
+
+      auto elem_segment_field = std::make_unique<ElemSegmentModuleField>(loc);
+      ElemSegment& elem_segment = elem_segment_field->elem_segment;
+      elem_segment.table_var = Var(module->tables.size(), GetLocation());
+      auto offset = table.elem_limits.is_64 ? Const::I64(0) : Const::I32(0);
+      elem_segment.offset.push_back(std::make_unique<ConstExpr>(offset));
+      elem_segment.offset.back().loc = loc;
+      elem_segment.elem_type = elem_type;
+      // Syntax is either an optional list of var (legacy), or a non-empty list
+      // of elem expr.
+      ExprList elem_expr;
+      if (ParseElemExprOpt(&elem_expr)) {
+        elem_segment.elem_exprs.push_back(std::move(elem_expr));
+        // Parse the rest.
+        ParseElemExprListOpt(&elem_segment.elem_exprs);
+      } else {
+        ParseElemExprVarListOpt(&elem_segment.elem_exprs);
+      }
+      EXPECT(Rpar);
+
+      table.elem_limits.initial = elem_segment.elem_exprs.size();
+      table.elem_limits.max = elem_segment.elem_exprs.size();
+      table.elem_limits.has_max = true;
+      table.elem_type = elem_type;
+      module->AppendField(std::move(field));
+      module->AppendField(std::move(elem_segment_field));
+    } else {
+      CHECK_RESULT(ParseLimits(&table.elem_limits));
+      CHECK_RESULT(ParseRefType(&table.elem_type));
+      module->AppendField(std::move(field));
+    }
   }
 
   AppendInlineExportFields(module, &export_fields, module->tables.size() - 1);
@@ -2071,11 +2194,11 @@ Result WastParser::ParseMemoryExpr(Location loc,
 template <typename T>
 Result WastParser::ParseMemoryBinaryExpr(Location loc,
                                          std::unique_ptr<Expr>* out_expr) {
-  Var srcmemidx;
   Var destmemidx;
-  CHECK_RESULT(ParseMemidx(loc, &srcmemidx));
+  Var srcmemidx;
   CHECK_RESULT(ParseMemidx(loc, &destmemidx));
-  out_expr->reset(new T(srcmemidx, destmemidx, loc));
+  CHECK_RESULT(ParseMemidx(loc, &srcmemidx));
+  out_expr->reset(new T(destmemidx, srcmemidx, loc));
   return Result::Ok;
 }
 
@@ -2707,7 +2830,8 @@ Result WastParser::ParseConst(Const* const_, ConstType const_type) {
     case Opcode::I32Const: {
       auto token = Consume();
       if (!token.HasLiteral()) {
-        return Result::Error;
+        result = Result::Error;
+        break;
       }
       auto sv = token.literal().text;
       uint32_t u32;
@@ -2719,7 +2843,8 @@ Result WastParser::ParseConst(Const* const_, ConstType const_type) {
     case Opcode::I64Const: {
       auto token = Consume();
       if (!token.HasLiteral()) {
-        return Result::Error;
+        result = Result::Error;
+        break;
       }
       auto sv = token.literal().text;
       uint64_t u64;
@@ -3022,36 +3147,27 @@ Result WastParser::ParseExpr(ExprList* exprs) {
         CHECK_RESULT(ParseLabelOpt(&expr->true_.label));
         CHECK_RESULT(ParseBlockDeclaration(&expr->true_.decl));
 
-        if (PeekMatchExpr()) {
+        while (PeekMatchExpr()) {
           ExprList cond;
           CHECK_RESULT(ParseExpr(&cond));
           exprs->splice(exprs->end(), cond);
         }
 
-        if (MatchLpar(TokenType::Then)) {
+        EXPECT(Lpar);
-          CHECK_RESULT(ParseTerminatingInstrList(&expr->true_.exprs));
+        if (!Match(TokenType::Then)) {
-          expr->true_.end_loc = GetLocation();
-          EXPECT(Rpar);
-
-          if (MatchLpar(TokenType::Else)) {
-            CHECK_RESULT(ParseTerminatingInstrList(&expr->false_));
-            EXPECT(Rpar);
-          } else if (PeekMatchExpr()) {
-            CHECK_RESULT(ParseExpr(&expr->false_));
-          }
-          expr->false_end_loc = GetLocation();
-        } else if (PeekMatchExpr()) {
-          CHECK_RESULT(ParseExpr(&expr->true_.exprs));
-          expr->true_.end_loc = GetLocation();
-          if (PeekMatchExpr()) {
-            CHECK_RESULT(ParseExpr(&expr->false_));
-            expr->false_end_loc = GetLocation();
-          }
-        } else {
-          ConsumeIfLpar();
           return ErrorExpected({"then block"}, "(then ...)");
         }
 
+        CHECK_RESULT(ParseTerminatingInstrList(&expr->true_.exprs));
+        expr->true_.end_loc = GetLocation();
+        EXPECT(Rpar);
+
+        if (MatchLpar(TokenType::Else)) {
+          CHECK_RESULT(ParseTerminatingInstrList(&expr->false_));
+          EXPECT(Rpar);
+        }
+        expr->false_end_loc = GetLocation();
+
         exprs->push_back(std::move(expr));
         break;
       }
@@ -3343,7 +3459,6 @@ Result WastParser::ParseModuleCommand(Script* script, CommandPtr* out_command) {
       module->name = bsm->name;
       module->loc = bsm->loc;
       for (const auto& error : errors) {
-        assert(error.error_level == ErrorLevel::Error);
         if (error.loc.offset == kInvalidOffset) {
           Error(bsm->loc, "error in binary module: %s", error.message.c_str());
         } else {
@@ -3358,7 +3473,26 @@ Result WastParser::ParseModuleCommand(Script* script, CommandPtr* out_command) {
     }
 
     case ScriptModuleType::Quoted:
-      return ErrorExpected({"a binary module", "a text module"});
+      auto command = std::make_unique<ModuleCommand>();
+      module = &command->module;
+      auto* qsm = cast<QuotedScriptModule>(script_module.get());
+      Errors errors;
+      const char* filename = "<text>";
+      std::unique_ptr<Module> m;
+      std::unique_ptr<WastLexer> lexer = WastLexer::CreateBufferLexer(
+          filename, qsm->data.data(), qsm->data.size(), &errors);
+      ParseWatModule(lexer.get(), &m, &errors, options_);
+      for (const auto& error : errors) {
+        if (error.loc.offset == kInvalidOffset) {
+          Error(qsm->loc, "error in quoted module: %s", error.message.c_str());
+        } else {
+          Error(qsm->loc, "error in quoted module: @0x%08" PRIzx ": %s",
+                error.loc.offset, error.message.c_str());
+        }
+      }
+      *module = std::move(*m.get());
+      *out_command = std::move(command);
+      break;
   }
 
   // script is nullptr when ParseModuleCommand is called from ParseModule.
@@ -3523,7 +3657,7 @@ Result WastParser::ParseScriptModule(
       auto tsm = std::make_unique<TextScriptModule>();
       tsm->module.name = name;
       tsm->module.loc = loc;
-      if (IsModuleField(PeekPair())) {
+      if (IsModuleField(PeekPair()) || PeekIsCustom()) {
         CHECK_RESULT(ParseModuleFieldList(&tsm->module));
       } else if (!PeekMatch(TokenType::Rpar)) {
         ConsumeIfLpar();
@@ -3590,6 +3724,43 @@ void WastParser::CheckImportOrdering(Module* module) {
   }
 }
 
+bool WastParser::HasError() const {
+  return std::any_of(errors_->begin(), errors_->end(), [](const auto& x) {
+    return x.error_level == ErrorLevel::Error;
+  });
+}
+
+void WastParser::TokenQueue::push_back(Token t) {
+  assert(!tokens[!i]);
+  tokens[!i] = t;
+  if (empty()) {
+    i = !i;
+  }
+}
+
+void WastParser::TokenQueue::pop_front() {
+  assert(tokens[i]);
+  tokens[i].reset();
+  i = !i;
+}
+
+const Token& WastParser::TokenQueue::at(size_t n) const {
+  assert(n <= 1);
+  return tokens[i ^ static_cast<bool>(n)].value();
+}
+
+const Token& WastParser::TokenQueue::front() const {
+  return at(0);
+}
+
+bool WastParser::TokenQueue::empty() const {
+  return !tokens[i];
+}
+
+size_t WastParser::TokenQueue::size() const {
+  return empty() ? 0 : 1 + tokens[!i].has_value();
+}
+
 Result ParseWatModule(WastLexer* lexer,
                       std::unique_ptr<Module>* out_module,
                       Errors* errors,

third_party/wasm2c/src/wat-writer.cc

@@ -164,6 +164,7 @@ class WatWriter : ModuleContext {
   void WriteTypeEntry(const TypeEntry& type);
   void WriteField(const Field& field);
   void WriteStartFunction(const Var& start);
+  void WriteCustom(const Custom& custom);
 
   class ExprVisitorDelegate;
 
@@ -671,7 +672,11 @@ Result WatWriter::ExprVisitorDelegate::OnCallIndirectExpr(
   writer_->WritePutsSpace(Opcode::CallIndirect_Opcode.GetName());
   writer_->WriteVarUnlessZero(expr->table, NextChar::Space);
   writer_->WriteOpenSpace("type");
-  writer_->WriteVar(expr->decl.type_var, NextChar::Newline);
+  const auto type_var =
+      expr->decl.has_func_type
+          ? expr->decl.type_var
+          : Var{writer_->module.GetFuncTypeIndex(expr->decl), expr->loc};
+  writer_->WriteVar(type_var, NextChar::Newline);
   writer_->WriteCloseNewline();
   return Result::Ok;
 }
@@ -780,7 +785,7 @@ Result WatWriter::ExprVisitorDelegate::EndLoopExpr(LoopExpr* expr) {
 
 Result WatWriter::ExprVisitorDelegate::OnMemoryCopyExpr(MemoryCopyExpr* expr) {
   writer_->WritePutsSpace(Opcode::MemoryCopy_Opcode.GetName());
-  writer_->WriteTwoMemoryVarsUnlessBothZero(expr->srcmemidx, expr->destmemidx,
+  writer_->WriteTwoMemoryVarsUnlessBothZero(expr->destmemidx, expr->srcmemidx,
                                             NextChar::Space);
   writer_->WriteNewline(NO_FORCE_NEWLINE);
   return Result::Ok;
@@ -911,7 +916,11 @@ Result WatWriter::ExprVisitorDelegate::OnReturnCallIndirectExpr(
     ReturnCallIndirectExpr* expr) {
   writer_->WritePutsSpace(Opcode::ReturnCallIndirect_Opcode.GetName());
   writer_->WriteOpenSpace("type");
-  writer_->WriteVar(expr->decl.type_var, NextChar::Space);
+  const auto type_var =
+      expr->decl.has_func_type
+          ? expr->decl.type_var
+          : Var{writer_->module.GetFuncTypeIndex(expr->decl), expr->loc};
+  writer_->WriteVar(type_var, NextChar::Space);
   writer_->WriteCloseNewline();
   return Result::Ok;
 }
@@ -1487,6 +1496,11 @@ void WatWriter::WriteMemory(const Memory& memory) {
   WriteInlineExports(ExternalKind::Memory, memory_index_);
   WriteInlineImport(ExternalKind::Memory, memory_index_);
   WriteLimits(memory.page_limits);
+  if (memory.page_size != WABT_DEFAULT_PAGE_SIZE) {
+    WriteOpenSpace("pagesize");
+    Writef("%u", memory.page_size);
+    WriteCloseSpace();
+  }
   WriteCloseNewline();
   memory_index_++;
 }
@@ -1495,7 +1509,11 @@ void WatWriter::WriteDataSegment(const DataSegment& segment) {
   WriteOpenSpace("data");
   WriteNameOrIndex(segment.name, data_segment_index_, NextChar::Space);
   if (segment.kind != SegmentKind::Passive) {
-    WriteMemoryVarUnlessZero(segment.memory_var, NextChar::Space);
+    if (module.GetMemoryIndex(segment.memory_var) != 0) {
+      WriteOpenSpace("memory");
+      WriteVar(segment.memory_var, NextChar::Space);
+      WriteCloseSpace();
+    }
     WriteInitExpr(segment.offset);
   }
   WriteQuotedData(segment.data.data(), segment.data.size());
@@ -1605,6 +1623,13 @@ void WatWriter::WriteStartFunction(const Var& start) {
   WriteCloseNewline();
 }
 
+void WatWriter::WriteCustom(const Custom& custom) {
+  WriteOpenSpace("@custom");
+  WriteQuotedString(custom.name, NextChar::Space);
+  WriteQuotedData(custom.data.data(), custom.data.size());
+  WriteCloseNewline();
+}
+
 Result WatWriter::WriteModule() {
   BuildInlineExportMap();
   BuildInlineImportMap();
@@ -1651,6 +1676,11 @@ Result WatWriter::WriteModule() {
         break;
     }
   }
+  if (options_.features.annotations_enabled()) {
+    for (const Custom& custom : module.customs) {
+      WriteCustom(custom);
+    }
+  }
   WriteCloseNewline();
   /* force the newline to be written */
   WriteNextChar();

third_party/wasm2c/wasm2c/.gitignore

@@ -1,4 +1,6 @@
 wasm-rt-impl.o
+wasm-rt-mem-impl.o
+wasm-rt-exceptions-impl.o
 examples/**/*.o
 examples/fac/fac
 examples/rot13/rot13
@@ -9,3 +11,7 @@ examples/callback/callback
 examples/callback/callback.c
 examples/callback/callback.h
 examples/callback/callback.wasm
+examples/threads/threads
+examples/threads/sample.c
+examples/threads/sample.h
+examples/threads/sample.wasm

third_party/wasm2c/wasm2c/README.md

@@ -11,6 +11,9 @@ $ wasm2c test.wasm -o test.c
 $ wasm2c test.wasm --no-debug-names -o test.c
 ```
 
+The C code produced targets the C99 standard. If, however, the Wasm module uses
+Wasm threads/atomics, the code produced targets the C11 standard.
+
 ## Tutorial: .wat -> .wasm -> .c
 
 Let's look at a simple example of a factorial function.
@@ -106,11 +109,11 @@ int main(int argc, char** argv) {
 ## Compiling the wasm2c output
 
 To compile the executable, we need to use `main.c` and the generated `fac.c`.
-We'll also include `wasm-rt-impl.c` which has implementations of the various
+We'll also include `wasm-rt-impl.c` and `wasm-rt-mem-impl.c`, which have implementations of the various
 `wasm_rt_*` functions used by `fac.c` and `fac.h`.
 
 ```sh
-$ cc -o fac main.c fac.c wasm-rt-impl.c
+$ cc -o fac main.c fac.c wasm2c/wasm-rt-impl.c wasm2c/wasm-rt-mem-impl.c -Iwasm2c -lm
 ```
 
 A note on compiling with optimization: wasm2c relies on certain
@@ -138,6 +141,52 @@ fac(10) -> 3628800
 You can take a look at the all of these files in
 [wasm2c/examples/fac](/wasm2c/examples/fac).
 
+### Enabling extra sanity checks
+
+Wasm2c provides a macro `WASM_RT_SANITY_CHECKS` that if defined enables
+additional sanity checks in the produced Wasm2c code. Note that this may have a
+high performance overhead, and is thus only recommended for debug builds.
+
+### Enabling Segue (a Linux x86_64 target specific optimization)
+
+Wasm2c can use the "Segue" optimization if allowed. The segue optimization uses
+an x86 segment register to store the location of Wasm's linear memory, when
+compiling a Wasm module with clang, running on x86_64 Linux, the macro
+`WASM_RT_ALLOW_SEGUE` is defined, and the flag `-mfsgsbase` is passed to clang.
+Segue is not used if
+
+1. The Wasm module uses a more than a single unshared imported or exported
+   memory
+2. The wasm2c code is compiled with GCC. Segue requires intrinsics for
+   (rd|wr)gsbase, "address namespaces" for accessing pointers, and support for
+   memcpy on pointers with custom "address namespaces". GCC does not support the
+   memcpy requirement.
+3. The code is compiled for Windows as Windows doesn't restore the segment
+   register on context switch.
+
+The wasm2c generated code automatically sets the unused segment register (the
+`%gs` register on x86_64 Linux) during the function calls into wasm2c generated
+module, restores it after calls to external modules etc. Any host function
+written in C would continue to work without changes as C code does not modify
+the unused segment register `%gs` (See
+[here](https://www.kernel.org/doc/html/next/x86/x86_64/fsgs.html) for details).
+However, any host functions written in assembly that clobber the free segment
+register must restore the value of this register prior to executing or returning
+control to wasm2c generated code.
+
+As an additional optimization, if the host program does not use the `%gs`
+segment register for any other purpose (which is typically the case in most
+programs), you can additionally allow wasm2c to unconditionally overwrite the
+value of the `%gs` register without restoring the old value. This can be done
+defining the macro `WASM_RT_SEGUE_FREE_SEGMENT`.
+
+You can test the performance of the Segue optimization by running Dhrystone with
+and without Segue:
+
+```bash
+cd wasm2c/benchmarks/segue && make
+```
+
 ## Looking at the generated header, `fac.h`
 
 The generated header file looks something like this:
@@ -255,11 +304,28 @@ specified by the module, or `0xffffffff` if there is no limit.
 ```c
 typedef struct {
   uint8_t* data;
-  uint32_t pages, max_pages;
+  uint64_t pages, max_pages;
-  uint32_t size;
+  uint64_t size;
+  bool is64;
 } wasm_rt_memory_t;
 ```
 
+This is followed by the definition of a shared memory instance. This is similar
+to a regular memory instance, but represents memory that can be used by multiple
+Wasm instances, and thus enforces a minimum amount of memory order on
+operations. The Shared memory definition has one additional member, `mem_lock`,
+which is a lock that is used during memory grow operations for thread safety.
+
+```c
+typedef struct {
+  _Atomic volatile uint8_t* data;
+  uint64_t pages, max_pages;
+  uint64_t size;
+  bool is64;
+  mtx_t mem_lock;
+} wasm_rt_shared_memory_t;
+```
+
 Next is the definition of a table instance. The `data` field is a pointer to
 `size` elements. Like a memory instance, `size` is the current size of a table,
 and `max_size` is the maximum size of the table, or `0xffffffff` if there is no
@@ -290,11 +356,16 @@ const char* wasm_rt_strerror(wasm_rt_trap_t trap);
 void wasm_rt_allocate_memory(wasm_rt_memory_t*, uint32_t initial_pages, uint32_t max_pages, bool is64);
 uint32_t wasm_rt_grow_memory(wasm_rt_memory_t*, uint32_t pages);
 void wasm_rt_free_memory(wasm_rt_memory_t*);
+void wasm_rt_allocate_memory_shared(wasm_rt_shared_memory_t*, uint32_t initial_pages, uint32_t max_pages, bool is64);
+uint32_t wasm_rt_grow_memory_shared(wasm_rt_shared_memory_t*, uint32_t pages);
+void wasm_rt_free_memory_shared(wasm_rt_shared_memory_t*);
 void wasm_rt_allocate_funcref_table(wasm_rt_table_t*, uint32_t elements, uint32_t max_elements);
 void wasm_rt_allocate_externref_table(wasm_rt_externref_table_t*, uint32_t elements, uint32_t max_elements);
 void wasm_rt_free_funcref_table(wasm_rt_table_t*);
 void wasm_rt_free_externref_table(wasm_rt_table_t*);
 uint32_t wasm_rt_call_stack_depth; /* on platforms that don't use the signal handler to detect exhaustion */
+void wasm_rt_init_thread(void);
+void wasm_rt_free_thread(void);
 ```
 
 `wasm_rt_init` must be called by the embedder before anything else, to
@@ -327,6 +398,16 @@ arguments and returning `void` . e.g.
 
 `wasm_rt_free_memory` frees the memory instance.
 
+`wasm_rt_allocate_memory_shared` initializes a memory instance that can be
+shared by different Wasm threads. It's operation is otherwise similar to
+`wasm_rt_allocate_memory`.
+
+`wasm_rt_grow_memory_shared` must grow the given shared memory instance by the
+given number of pages. It's operation is otherwise similar to
+`wasm_rt_grow_memory`.
+
+`wasm_rt_free_memory_shared` frees the shared memory instance.
+
 `wasm_rt_allocate_funcref_table` and the similar `..._externref_table`
 initialize a table instance of the given type, and allocate at least
 enough space for the given number of initial elements. The elements
@@ -339,6 +420,11 @@ shared between modules, it must be defined only once, by the embedder.
 It is only used on platforms that don't use the signal handler to detect
 exhaustion.
 
+`wasm_rt_init_thread` and `wasm_rt_free_thread` are used to initialize
+and free the runtime state for a given thread (other than the one that
+called `wasm_rt_init`). An example can be found in
+`wasm2c/examples/threads`.
+
 ### Runtime support for exception handling
 
 Several additional symbols must be defined if wasm2c is being run with support

third_party/wasm2c/wasm2c/wasm-rt-impl-tableops.inc

@@ -0,0 +1,78 @@
+/*
+ * Copyright 2018 WebAssembly Community Group participants
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// This file is used as a template to generate code for table operations for
+// funcref or externref. For this, the file must be included after defining
+// either WASM_RT_TABLE_OPS_FUNCREF or WASM_RT_TABLE_OPS_EXTERNREF
+
+#if defined(WASM_RT_TABLE_OPS_FUNCREF) && defined(WASM_RT_TABLE_OPS_EXTERNREF)
+#error \
+    "Expected only one of { WASM_RT_TABLE_OPS_FUNCREF, WASM_RT_TABLE_OPS_EXTERNREF } to be defined"
+#elif !defined(WASM_RT_TABLE_OPS_FUNCREF) && \
+    !defined(WASM_RT_TABLE_OPS_EXTERNREF)
+#error \
+    "Expected one of { WASM_RT_TABLE_OPS_FUNCREF, WASM_RT_TABLE_OPS_EXTERNREF } to be defined"
+#endif
+
+#ifdef WASM_RT_TABLE_OPS_FUNCREF
+#define WASM_RT_TABLE_TYPE wasm_rt_funcref_table_t
+#define WASM_RT_TABLE_ELEMENT_TYPE wasm_rt_funcref_t
+#define WASM_RT_TABLE_APINAME(name) name##_funcref_table
+#else
+#define WASM_RT_TABLE_TYPE wasm_rt_externref_table_t
+#define WASM_RT_TABLE_ELEMENT_TYPE wasm_rt_externref_t
+#define WASM_RT_TABLE_APINAME(name) name##_externref_table
+#endif
+
+void WASM_RT_TABLE_APINAME(wasm_rt_allocate)(WASM_RT_TABLE_TYPE* table,
+                                             uint32_t elements,
+                                             uint32_t max_elements) {
+  table->size = elements;
+  table->max_size = max_elements;
+  table->data = calloc(table->size, sizeof(WASM_RT_TABLE_ELEMENT_TYPE));
+}
+
+void WASM_RT_TABLE_APINAME(wasm_rt_free)(WASM_RT_TABLE_TYPE* table) {
+  free(table->data);
+}
+
+uint32_t WASM_RT_TABLE_APINAME(wasm_rt_grow)(WASM_RT_TABLE_TYPE* table,
+                                             uint32_t delta,
+                                             WASM_RT_TABLE_ELEMENT_TYPE init) {
+  uint32_t old_elems = table->size;
+  uint64_t new_elems = (uint64_t)table->size + delta;
+  if (new_elems == 0) {
+    return 0;
+  }
+  if ((new_elems < old_elems) || (new_elems > table->max_size)) {
+    return (uint32_t)-1;
+  }
+  void* new_data =
+      realloc(table->data, new_elems * sizeof(WASM_RT_TABLE_ELEMENT_TYPE));
+  if (!new_data) {
+    return (uint32_t)-1;
+  }
+  table->data = new_data;
+  table->size = new_elems;
+  for (uint32_t i = old_elems; i < new_elems; i++) {
+    table->data[i] = init;
+  }
+  return old_elems;
+}
+
+#undef WASM_RT_TABLE_APINAME
+#undef WASM_RT_TABLE_ELEMENT_TYPE
+#undef WASM_RT_TABLE_TYPE

third_party/wasm2c/wasm2c/wasm-rt-impl.c

@@ -36,35 +36,58 @@
 #include <sys/mman.h>
 #endif
 
-#define PAGE_SIZE 65536
+#ifndef NDEBUG
+#define DEBUG_PRINTF(...) fprintf(stderr, __VA_ARGS__);
+#else
+#define DEBUG_PRINTF(...)
+#endif
 
 #if WASM_RT_INSTALL_SIGNAL_HANDLER
 static bool g_signal_handler_installed = false;
 #ifdef _WIN32
 static void* g_sig_handler_handle = 0;
-#else
-static char* g_alt_stack = 0;
 #endif
 #endif
 
-#if WASM_RT_USE_STACK_DEPTH_COUNT
+#if WASM_RT_USE_SEGUE
+// Currently Segue is used only for linux
+#include <sys/auxv.h>
+#ifdef __GLIBC__
+#include <gnu/libc-version.h>
+#endif
+bool wasm_rt_fsgsbase_inst_supported = false;
+
+#include <asm/prctl.h>    // For ARCH_SET_GS
+#include <sys/syscall.h>  // For SYS_arch_prctl
+#include <unistd.h>       // For syscall
+
+#ifndef HWCAP2_FSGSBASE
+#define HWCAP2_FSGSBASE (1 << 1)
+#endif
+#endif
+
+#if WASM_RT_SEGUE_FREE_SEGMENT
+WASM_RT_THREAD_LOCAL void* wasm_rt_last_segment_val = NULL;
+#endif
+
+#if WASM_RT_STACK_DEPTH_COUNT
 WASM_RT_THREAD_LOCAL uint32_t wasm_rt_call_stack_depth;
 WASM_RT_THREAD_LOCAL uint32_t wasm_rt_saved_call_stack_depth;
+#elif WASM_RT_STACK_EXHAUSTION_HANDLER
+static WASM_RT_THREAD_LOCAL void* g_alt_stack = NULL;
 #endif
 
+#ifndef WASM_RT_TRAP_HANDLER
 WASM_RT_THREAD_LOCAL wasm_rt_jmp_buf g_wasm_rt_jmp_buf;
+#endif
 
 #ifdef WASM_RT_TRAP_HANDLER
 extern void WASM_RT_TRAP_HANDLER(wasm_rt_trap_t code);
 #endif
 
-#ifdef WASM_RT_GROW_FAILED_HANDLER
-extern void WASM_RT_GROW_FAILED_HANDLER();
-#endif
-
 void wasm_rt_trap(wasm_rt_trap_t code) {
   assert(code != WASM_RT_TRAP_NONE);
-#if WASM_RT_USE_STACK_DEPTH_COUNT
+#if WASM_RT_STACK_DEPTH_COUNT
   wasm_rt_call_stack_depth = wasm_rt_saved_call_stack_depth;
 #endif
 
@@ -77,47 +100,6 @@ void wasm_rt_trap(wasm_rt_trap_t code) {
 }
 
 #ifdef _WIN32
-static void* os_mmap(size_t size) {
-  void* ret = VirtualAlloc(NULL, size, MEM_RESERVE, PAGE_NOACCESS);
-  return ret;
-}
-
-static int os_munmap(void* addr, size_t size) {
-  // Windows can only unmap the whole mapping
-  (void)size; /* unused */
-  BOOL succeeded = VirtualFree(addr, 0, MEM_RELEASE);
-  return succeeded ? 0 : -1;
-}
-
-static int os_mprotect(void* addr, size_t size) {
-  if (size == 0) {
-    return 0;
-  }
-  void* ret = VirtualAlloc(addr, size, MEM_COMMIT, PAGE_READWRITE);
-  if (ret == addr) {
-    return 0;
-  }
-  VirtualFree(addr, 0, MEM_RELEASE);
-  return -1;
-}
-
-static void os_print_last_error(const char* msg) {
-  DWORD errorMessageID = GetLastError();
-  if (errorMessageID != 0) {
-    LPSTR messageBuffer = 0;
-    // The api creates the buffer that holds the message
-    size_t size = FormatMessageA(
-        FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM |
-            FORMAT_MESSAGE_IGNORE_INSERTS,
-        NULL, errorMessageID, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
-        (LPSTR)&messageBuffer, 0, NULL);
-    (void)size;
-    printf("%s. %s\n", msg, messageBuffer);
-    LocalFree(messageBuffer);
-  } else {
-    printf("%s. No error code.\n", msg);
-  }
-}
 
 #if WASM_RT_INSTALL_SIGNAL_HANDLER
 
@@ -142,26 +124,6 @@ static void os_cleanup_signal_handler(void) {
 #endif
 
 #else
-static void* os_mmap(size_t size) {
-  int map_prot = PROT_NONE;
-  int map_flags = MAP_ANONYMOUS | MAP_PRIVATE;
-  uint8_t* addr = mmap(NULL, size, map_prot, map_flags, -1, 0);
-  if (addr == MAP_FAILED)
-    return NULL;
-  return addr;
-}
-
-static int os_munmap(void* addr, size_t size) {
-  return munmap(addr, size);
-}
-
-static int os_mprotect(void* addr, size_t size) {
-  return mprotect(addr, size, PROT_READ | PROT_WRITE);
-}
-
-static void os_print_last_error(const char* msg) {
-  perror(msg);
-}
 
 #if WASM_RT_INSTALL_SIGNAL_HANDLER
 static void os_signal_handler(int sig, siginfo_t* si, void* unused) {
@@ -173,25 +135,12 @@ static void os_signal_handler(int sig, siginfo_t* si, void* unused) {
 }
 
 static void os_install_signal_handler(void) {
-  /* Use alt stack to handle SIGSEGV from stack overflow */
-  g_alt_stack = malloc(SIGSTKSZ);
-  if (g_alt_stack == NULL) {
-    perror("malloc failed");
-    abort();
-  }
-
-  stack_t ss;
-  ss.ss_sp = g_alt_stack;
-  ss.ss_flags = 0;
-  ss.ss_size = SIGSTKSZ;
-  if (sigaltstack(&ss, NULL) != 0) {
-    perror("sigaltstack failed");
-    abort();
-  }
-
   struct sigaction sa;
   memset(&sa, '\0', sizeof(sa));
-  sa.sa_flags = SA_SIGINFO | SA_ONSTACK;
+  sa.sa_flags = SA_SIGINFO;
+#if WASM_RT_STACK_EXHAUSTION_HANDLER
+  sa.sa_flags |= SA_ONSTACK;
+#endif
   sigemptyset(&sa.sa_mask);
   sa.sa_sigaction = os_signal_handler;
 
@@ -211,28 +160,108 @@ static void os_cleanup_signal_handler(void) {
     perror("sigaction failed");
     abort();
   }
+}
+#endif
 
-  if (sigaltstack(NULL, NULL) != 0) {
+#if WASM_RT_STACK_EXHAUSTION_HANDLER
+static bool os_has_altstack_installed() {
+  /* check for altstack already in place */
+  stack_t ss;
+  if (sigaltstack(NULL, &ss) != 0) {
     perror("sigaltstack failed");
     abort();
   }
 
+  return !(ss.ss_flags & SS_DISABLE);
+}
+
+/* These routines set up an altstack to handle SIGSEGV from stack overflow. */
+static void os_allocate_and_install_altstack(void) {
+  /* verify altstack not already allocated */
+  assert(!g_alt_stack &&
+         "wasm-rt error: tried to re-allocate thread-local alternate stack");
+
+  /* We could check and warn if an altstack is already installed, but some
+   * sanitizers install their own altstack, so this warning would fire
+   * spuriously and break the test outputs. */
+
+  /* allocate altstack */
+  g_alt_stack = malloc(SIGSTKSZ);
+  if (g_alt_stack == NULL) {
+    perror("malloc failed");
+    abort();
+  }
+
+  /* install altstack */
+  stack_t ss;
+  ss.ss_sp = g_alt_stack;
+  ss.ss_flags = 0;
+  ss.ss_size = SIGSTKSZ;
+  if (sigaltstack(&ss, NULL) != 0) {
+    perror("sigaltstack failed");
+    abort();
+  }
+}
+
+static void os_disable_and_deallocate_altstack(void) {
+  /* in debug build, verify altstack allocated */
+  assert(g_alt_stack &&
+         "wasm-rt error: thread-local alternate stack not allocated");
+
+  /* verify altstack was still in place */
+  stack_t ss;
+  if (sigaltstack(NULL, &ss) != 0) {
+    perror("sigaltstack failed");
+    abort();
+  }
+
+  if ((!g_alt_stack) || (ss.ss_flags & SS_DISABLE) ||
+      (ss.ss_sp != g_alt_stack) || (ss.ss_size != SIGSTKSZ)) {
+    DEBUG_PRINTF(
+        "wasm-rt warning: alternate stack was modified unexpectedly\n");
+    return;
+  }
+
+  /* disable and free */
+  ss.ss_flags = SS_DISABLE;
+  if (sigaltstack(&ss, NULL) != 0) {
+    perror("sigaltstack failed");
+    abort();
+  }
+  assert(!os_has_altstack_installed());
   free(g_alt_stack);
+  g_alt_stack = NULL;
 }
 #endif
 
 #endif
 
 void wasm_rt_init(void) {
+  wasm_rt_init_thread();
 #if WASM_RT_INSTALL_SIGNAL_HANDLER
   if (!g_signal_handler_installed) {
     g_signal_handler_installed = true;
     os_install_signal_handler();
   }
 #endif
+
+#if WASM_RT_USE_SEGUE
+#if defined(__GLIBC__) && __GLIBC__ >= 2 && __GLIBC_MINOR__ >= 18
+  // Check for support for userspace wrgsbase instructions
+  unsigned long val = getauxval(AT_HWCAP2);
+  wasm_rt_fsgsbase_inst_supported = val & HWCAP2_FSGSBASE;
+#endif
+#endif
+
+  assert(wasm_rt_is_initialized());
 }
 
 bool wasm_rt_is_initialized(void) {
+#if WASM_RT_STACK_EXHAUSTION_HANDLER
+  if (!os_has_altstack_installed()) {
+    return false;
+  }
+#endif
 #if WASM_RT_INSTALL_SIGNAL_HANDLER
   return g_signal_handler_installed;
 #else
@@ -241,156 +270,52 @@ bool wasm_rt_is_initialized(void) {
 }
 
 void wasm_rt_free(void) {
+  assert(wasm_rt_is_initialized());
 #if WASM_RT_INSTALL_SIGNAL_HANDLER
   os_cleanup_signal_handler();
   g_signal_handler_installed = false;
 #endif
+  wasm_rt_free_thread();
 }
 
-#if WASM_RT_USE_MMAP
+void wasm_rt_init_thread(void) {
-
+#if WASM_RT_STACK_EXHAUSTION_HANDLER
-static uint64_t get_allocation_size_for_mmap(wasm_rt_memory_t* memory) {
+  os_allocate_and_install_altstack();
-  assert(!memory->is64 &&
-         "memory64 is not yet compatible with WASM_RT_USE_MMAP");
-#if WASM_RT_MEMCHECK_GUARD_PAGES
-  /* Reserve 8GiB. */
-  const uint64_t max_size = 0x200000000ul;
-  return max_size;
-#else
-  if (memory->max_pages != 0) {
-    const uint64_t max_size = memory->max_pages * PAGE_SIZE;
-    return max_size;
-  }
-
-  /* Reserve 4GiB. */
-  const uint64_t max_size = 0x100000000ul;
-  return max_size;
 #endif
 }
 
+void wasm_rt_free_thread(void) {
+#if WASM_RT_STACK_EXHAUSTION_HANDLER
+  os_disable_and_deallocate_altstack();
 #endif
+}
 
-void wasm_rt_allocate_memory(wasm_rt_memory_t* memory,
+#if WASM_RT_USE_SEGUE
-                             uint64_t initial_pages,
+void wasm_rt_syscall_set_segue_base(void* base) {
-                             uint64_t max_pages,
+  if (syscall(SYS_arch_prctl, ARCH_SET_GS, base) != 0) {
-                             bool is64) {
+    perror("wasm_rt_syscall_set_segue_base error");
-  uint64_t byte_length = initial_pages * PAGE_SIZE;
-  memory->size = byte_length;
-  memory->pages = initial_pages;
-  memory->max_pages = max_pages;
-  memory->is64 = is64;
-
-#if WASM_RT_USE_MMAP
-  const uint64_t mmap_size = get_allocation_size_for_mmap(memory);
-  void* addr = os_mmap(mmap_size);
-  if (!addr) {
-    os_print_last_error("os_mmap failed.");
     abort();
   }
-  int ret = os_mprotect(addr, byte_length);
+}
-  if (ret != 0) {
+void* wasm_rt_syscall_get_segue_base() {
-    os_print_last_error("os_mprotect failed.");
+  void* base;
+  if (syscall(SYS_arch_prctl, ARCH_GET_GS, &base) != 0) {
+    perror("wasm_rt_syscall_get_segue_base error");
     abort();
   }
-  memory->data = addr;
+  return base;
-#else
-  memory->data = calloc(byte_length, 1);
-#endif
 }
-
-static uint64_t grow_memory_impl(wasm_rt_memory_t* memory, uint64_t delta) {
-  uint64_t old_pages = memory->pages;
-  uint64_t new_pages = memory->pages + delta;
-  if (new_pages == 0) {
-    return 0;
-  }
-  if (new_pages < old_pages || new_pages > memory->max_pages) {
-    return (uint64_t)-1;
-  }
-  uint64_t old_size = old_pages * PAGE_SIZE;
-  uint64_t new_size = new_pages * PAGE_SIZE;
-  uint64_t delta_size = delta * PAGE_SIZE;
-#if WASM_RT_USE_MMAP
-  uint8_t* new_data = memory->data;
-  int ret = os_mprotect(new_data + old_size, delta_size);
-  if (ret != 0) {
-    return (uint64_t)-1;
-  }
-#else
-  uint8_t* new_data = realloc(memory->data, new_size);
-  if (new_data == NULL) {
-    return (uint64_t)-1;
-  }
-#if !WABT_BIG_ENDIAN
-  memset(new_data + old_size, 0, delta_size);
 #endif
-#endif
-#if WABT_BIG_ENDIAN
-  memmove(new_data + new_size - old_size, new_data, old_size);
-  memset(new_data, 0, delta_size);
-#endif
-  memory->pages = new_pages;
-  memory->size = new_size;
-  memory->data = new_data;
-  return old_pages;
-}
 
-uint64_t wasm_rt_grow_memory(wasm_rt_memory_t* memory, uint64_t delta) {
+// Include table operations for funcref
-  uint64_t ret = grow_memory_impl(memory, delta);
+#define WASM_RT_TABLE_OPS_FUNCREF
-#ifdef WASM_RT_GROW_FAILED_HANDLER
+#include "wasm-rt-impl-tableops.inc"
-  if (ret == -1) {
+#undef WASM_RT_TABLE_OPS_FUNCREF
-    WASM_RT_GROW_FAILED_HANDLER();
-  }
-#endif
-  return ret;
-}
 
-void wasm_rt_free_memory(wasm_rt_memory_t* memory) {
+// Include table operations for externref
-#if WASM_RT_USE_MMAP
+#define WASM_RT_TABLE_OPS_EXTERNREF
-  const uint64_t mmap_size = get_allocation_size_for_mmap(memory);
+#include "wasm-rt-impl-tableops.inc"
-  os_munmap(memory->data, mmap_size);  // ignore error
+#undef WASM_RT_TABLE_OPS_EXTERNREF
-#else
-  free(memory->data);
-#endif
-}
-
-#define DEFINE_TABLE_OPS(type)                                          \
-  void wasm_rt_allocate_##type##_table(wasm_rt_##type##_table_t* table, \
-                                       uint32_t elements,               \
-                                       uint32_t max_elements) {         \
-    table->size = elements;                                             \
-    table->max_size = max_elements;                                     \
-    table->data = calloc(table->size, sizeof(wasm_rt_##type##_t));      \
-  }                                                                     \
-  void wasm_rt_free_##type##_table(wasm_rt_##type##_table_t* table) {   \
-    free(table->data);                                                  \
-  }                                                                     \
-  uint32_t wasm_rt_grow_##type##_table(wasm_rt_##type##_table_t* table, \
-                                       uint32_t delta,                  \
-                                       wasm_rt_##type##_t init) {       \
-    uint32_t old_elems = table->size;                                   \
-    uint64_t new_elems = (uint64_t)table->size + delta;                 \
-    if (new_elems == 0) {                                               \
-      return 0;                                                         \
-    }                                                                   \
-    if ((new_elems < old_elems) || (new_elems > table->max_size)) {     \
-      return (uint32_t)-1;                                              \
-    }                                                                   \
-    void* new_data =                                                    \
-        realloc(table->data, new_elems * sizeof(wasm_rt_##type##_t));   \
-    if (!new_data) {                                                    \
-      return (uint32_t)-1;                                              \
-    }                                                                   \
-    table->data = new_data;                                             \
-    table->size = new_elems;                                            \
-    for (uint32_t i = old_elems; i < new_elems; i++) {                  \
-      table->data[i] = init;                                            \
-    }                                                                   \
-    return old_elems;                                                   \
-  }
-
-DEFINE_TABLE_OPS(funcref)
-DEFINE_TABLE_OPS(externref)
 
 const char* wasm_rt_strerror(wasm_rt_trap_t trap) {
   switch (trap) {
@@ -414,9 +339,11 @@ const char* wasm_rt_strerror(wasm_rt_trap_t trap) {
     case WASM_RT_TRAP_UNREACHABLE:
       return "Unreachable instruction executed";
     case WASM_RT_TRAP_CALL_INDIRECT:
-      return "Invalid call_indirect";
+      return "Invalid call_indirect or return_call_indirect";
     case WASM_RT_TRAP_UNCAUGHT_EXCEPTION:
       return "Uncaught exception";
+    case WASM_RT_TRAP_UNALIGNED:
+      return "Unaligned atomic memory access";
   }
   return "invalid trap code";
 }

third_party/wasm2c/wasm2c/wasm-rt-impl.h

@@ -27,10 +27,12 @@
 extern "C" {
 #endif
 
+#ifndef WASM_RT_TRAP_HANDLER
 /** A setjmp buffer used for handling traps. */
 extern WASM_RT_THREAD_LOCAL wasm_rt_jmp_buf g_wasm_rt_jmp_buf;
+#endif
 
-#if WASM_RT_USE_STACK_DEPTH_COUNT
+#if WASM_RT_STACK_DEPTH_COUNT
 /** Saved call stack depth that will be restored in case a trap occurs. */
 extern WASM_RT_THREAD_LOCAL uint32_t wasm_rt_saved_call_stack_depth;
 #define WASM_RT_SAVE_STACK_DEPTH() \

third_party/wasm2c/wasm2c/wasm-rt-mem-impl-helper.inc

@@ -0,0 +1,164 @@
+/*
+ * Copyright 2018 WebAssembly Community Group participants
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// This file is used as a template to generate code for regular memories or for
+// shared memories. For this, the file must be included after defining either
+// WASM_RT_MEM_OPS or WASM_RT_MEM_OPS_SHARED.
+
+#if defined(WASM_RT_MEM_OPS) && defined(WASM_RT_MEM_OPS_SHARED)
+#error \
+    "Expected only one of { WASM_RT_MEM_OPS, WASM_RT_MEM_OPS_SHARED } to be defined"
+#elif !defined(WASM_RT_MEM_OPS) && !defined(WASM_RT_MEM_OPS_SHARED)
+#error \
+    "Expected one of { WASM_RT_MEM_OPS, WASM_RT_MEM_OPS_SHARED } to be defined"
+#endif
+
+// Shared memory operations are defined only if we have C11
+#if defined(WASM_RT_MEM_OPS) || \
+    (defined(WASM_RT_MEM_OPS_SHARED) && defined(WASM_RT_C11_AVAILABLE))
+
+#ifdef WASM_RT_MEM_OPS
+
+// Memory operations on wasm_rt_memory_t
+#define MEMORY_TYPE wasm_rt_memory_t
+#define MEMORY_API_NAME(name) name
+#define MEMORY_CELL_TYPE uint8_t*
+#define MEMORY_LOCK_VAR_INIT(name)
+#define MEMORY_LOCK_AQUIRE(name)
+#define MEMORY_LOCK_RELEASE(name)
+
+#else
+
+// Memory operations on wasm_rt_shared_memory_t
+#define MEMORY_TYPE wasm_rt_shared_memory_t
+#define MEMORY_API_NAME(name) name##_shared
+#define MEMORY_CELL_TYPE _Atomic volatile uint8_t*
+
+#if WASM_RT_USE_C11THREADS
+#define MEMORY_LOCK_VAR_INIT(name) C11_MEMORY_LOCK_VAR_INIT(name)
+#define MEMORY_LOCK_AQUIRE(name) C11_MEMORY_LOCK_AQUIRE(name)
+#define MEMORY_LOCK_RELEASE(name) C11_MEMORY_LOCK_RELEASE(name)
+#elif WASM_RT_USE_PTHREADS
+#define MEMORY_LOCK_VAR_INIT(name) PTHREAD_MEMORY_LOCK_VAR_INIT(name)
+#define MEMORY_LOCK_AQUIRE(name) PTHREAD_MEMORY_LOCK_AQUIRE(name)
+#define MEMORY_LOCK_RELEASE(name) PTHREAD_MEMORY_LOCK_RELEASE(name)
+#elif WASM_RT_USE_CRITICALSECTION
+#define MEMORY_LOCK_VAR_INIT(name) WIN_MEMORY_LOCK_VAR_INIT(name)
+#define MEMORY_LOCK_AQUIRE(name) WIN_MEMORY_LOCK_AQUIRE(name)
+#define MEMORY_LOCK_RELEASE(name) WIN_MEMORY_LOCK_RELEASE(name)
+#endif
+
+#endif
+
+void MEMORY_API_NAME(wasm_rt_allocate_memory)(MEMORY_TYPE* memory,
+                                              uint64_t initial_pages,
+                                              uint64_t max_pages,
+                                              bool is64) {
+  uint64_t byte_length = initial_pages * WASM_PAGE_SIZE;
+  memory->size = byte_length;
+  memory->pages = initial_pages;
+  memory->max_pages = max_pages;
+  memory->is64 = is64;
+  MEMORY_LOCK_VAR_INIT(memory->mem_lock);
+
+#if WASM_RT_USE_MMAP
+  const uint64_t mmap_size =
+      get_alloc_size_for_mmap(memory->max_pages, memory->is64);
+  void* addr = os_mmap(mmap_size);
+  if (!addr) {
+    os_print_last_error("os_mmap failed.");
+    abort();
+  }
+  int ret = os_mprotect(addr, byte_length);
+  if (ret != 0) {
+    os_print_last_error("os_mprotect failed.");
+    abort();
+  }
+  memory->data = addr;
+#else
+  memory->data = calloc(byte_length, 1);
+#endif
+}
+
+static uint64_t MEMORY_API_NAME(grow_memory_impl)(MEMORY_TYPE* memory,
+                                                  uint64_t delta) {
+  uint64_t old_pages = memory->pages;
+  uint64_t new_pages = memory->pages + delta;
+  if (new_pages == 0) {
+    return 0;
+  }
+  if (new_pages < old_pages || new_pages > memory->max_pages) {
+    return (uint64_t)-1;
+  }
+  uint64_t old_size = old_pages * WASM_PAGE_SIZE;
+  uint64_t new_size = new_pages * WASM_PAGE_SIZE;
+  uint64_t delta_size = delta * WASM_PAGE_SIZE;
+#if WASM_RT_USE_MMAP
+  MEMORY_CELL_TYPE new_data = memory->data;
+  int ret = os_mprotect((void*)(new_data + old_size), delta_size);
+  if (ret != 0) {
+    return (uint64_t)-1;
+  }
+#else
+  MEMORY_CELL_TYPE new_data = realloc((void*)memory->data, new_size);
+  if (new_data == NULL) {
+    return (uint64_t)-1;
+  }
+#if !WABT_BIG_ENDIAN
+  memset((void*)(new_data + old_size), 0, delta_size);
+#endif
+#endif
+#if WABT_BIG_ENDIAN
+  memmove((void*)(new_data + new_size - old_size), (void*)new_data, old_size);
+  memset((void*)new_data, 0, delta_size);
+#endif
+  memory->pages = new_pages;
+  memory->size = new_size;
+  memory->data = new_data;
+  return old_pages;
+}
+
+uint64_t MEMORY_API_NAME(wasm_rt_grow_memory)(MEMORY_TYPE* memory,
+                                              uint64_t delta) {
+  MEMORY_LOCK_AQUIRE(memory->mem_lock);
+  uint64_t ret = MEMORY_API_NAME(grow_memory_impl)(memory, delta);
+  MEMORY_LOCK_RELEASE(memory->mem_lock);
+#ifdef WASM_RT_GROW_FAILED_HANDLER
+  if (ret == (uint64_t)-1) {
+    WASM_RT_GROW_FAILED_HANDLER();
+  }
+#endif
+  return ret;
+}
+
+void MEMORY_API_NAME(wasm_rt_free_memory)(MEMORY_TYPE* memory) {
+#if WASM_RT_USE_MMAP
+  const uint64_t mmap_size =
+      get_alloc_size_for_mmap(memory->max_pages, memory->is64);
+  os_munmap((void*)memory->data, mmap_size);  // ignore error
+#else
+  free((void*)memory->data);
+#endif
+}
+
+#undef MEMORY_LOCK_RELEASE
+#undef MEMORY_LOCK_AQUIRE
+#undef MEMORY_LOCK_VAR_INIT
+#undef MEMORY_CELL_TYPE
+#undef MEMORY_API_NAME
+#undef MEMORY_TYPE
+
+#endif

third_party/wasm2c/wasm2c/wasm-rt-mem-impl.c

@@ -0,0 +1,178 @@
+/*
+ * Copyright 2018 WebAssembly Community Group participants
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "wasm-rt-impl.h"
+
+#include <assert.h>
+#include <stdio.h>
+
+#ifdef _WIN32
+#include <windows.h>
+#else
+#include <sys/mman.h>
+#endif
+
+#define WASM_PAGE_SIZE 65536
+
+#ifdef WASM_RT_GROW_FAILED_HANDLER
+extern void WASM_RT_GROW_FAILED_HANDLER();
+#endif
+
+#define C11_MEMORY_LOCK_VAR_INIT(name)                \
+  if (mtx_init(&(name), mtx_plain) != thrd_success) { \
+    fprintf(stderr, "Lock init failed\n");            \
+    abort();                                          \
+  }
+#define C11_MEMORY_LOCK_AQUIRE(name)          \
+  if (mtx_lock(&(name)) != thrd_success) {    \
+    fprintf(stderr, "Lock acquire failed\n"); \
+    abort();                                  \
+  }
+#define C11_MEMORY_LOCK_RELEASE(name)         \
+  if (mtx_unlock(&(name)) != thrd_success) {  \
+    fprintf(stderr, "Lock release failed\n"); \
+    abort();                                  \
+  }
+
+#define PTHREAD_MEMORY_LOCK_VAR_INIT(name)      \
+  if (pthread_mutex_init(&(name), NULL) != 0) { \
+    fprintf(stderr, "Lock init failed\n");      \
+    abort();                                    \
+  }
+#define PTHREAD_MEMORY_LOCK_AQUIRE(name)      \
+  if (pthread_mutex_lock(&(name)) != 0) {     \
+    fprintf(stderr, "Lock acquire failed\n"); \
+    abort();                                  \
+  }
+#define PTHREAD_MEMORY_LOCK_RELEASE(name)     \
+  if (pthread_mutex_unlock(&(name)) != 0) {   \
+    fprintf(stderr, "Lock release failed\n"); \
+    abort();                                  \
+  }
+
+#define WIN_MEMORY_LOCK_VAR_INIT(name) InitializeCriticalSection(&(name))
+#define WIN_MEMORY_LOCK_AQUIRE(name) EnterCriticalSection(&(name))
+#define WIN_MEMORY_LOCK_RELEASE(name) LeaveCriticalSection(&(name))
+
+#if WASM_RT_USE_MMAP
+
+#ifdef _WIN32
+static void* os_mmap(size_t size) {
+  void* ret = VirtualAlloc(NULL, size, MEM_RESERVE, PAGE_NOACCESS);
+  return ret;
+}
+
+static int os_munmap(void* addr, size_t size) {
+  // Windows can only unmap the whole mapping
+  (void)size; /* unused */
+  BOOL succeeded = VirtualFree(addr, 0, MEM_RELEASE);
+  return succeeded ? 0 : -1;
+}
+
+static int os_mprotect(void* addr, size_t size) {
+  if (size == 0) {
+    return 0;
+  }
+  void* ret = VirtualAlloc(addr, size, MEM_COMMIT, PAGE_READWRITE);
+  if (ret == addr) {
+    return 0;
+  }
+  VirtualFree(addr, 0, MEM_RELEASE);
+  return -1;
+}
+
+static void os_print_last_error(const char* msg) {
+  DWORD errorMessageID = GetLastError();
+  if (errorMessageID != 0) {
+    LPSTR messageBuffer = 0;
+    // The api creates the buffer that holds the message
+    size_t size = FormatMessageA(
+        FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM |
+            FORMAT_MESSAGE_IGNORE_INSERTS,
+        NULL, errorMessageID, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
+        (LPSTR)&messageBuffer, 0, NULL);
+    (void)size;
+    printf("%s. %s\n", msg, messageBuffer);
+    LocalFree(messageBuffer);
+  } else {
+    printf("%s. No error code.\n", msg);
+  }
+}
+
+#else
+static void* os_mmap(size_t size) {
+  int map_prot = PROT_NONE;
+  int map_flags = MAP_ANONYMOUS | MAP_PRIVATE;
+  uint8_t* addr = mmap(NULL, size, map_prot, map_flags, -1, 0);
+  if (addr == MAP_FAILED)
+    return NULL;
+  return addr;
+}
+
+static int os_munmap(void* addr, size_t size) {
+  return munmap(addr, size);
+}
+
+static int os_mprotect(void* addr, size_t size) {
+  return mprotect(addr, size, PROT_READ | PROT_WRITE);
+}
+
+static void os_print_last_error(const char* msg) {
+  perror(msg);
+}
+
+#endif
+
+static uint64_t get_alloc_size_for_mmap(uint64_t max_pages, bool is64) {
+  assert(!is64 && "memory64 is not yet compatible with WASM_RT_USE_MMAP");
+#if WASM_RT_MEMCHECK_GUARD_PAGES
+  /* Reserve 8GiB. */
+  const uint64_t max_size = 0x200000000ul;
+  return max_size;
+#else
+  if (max_pages != 0) {
+    const uint64_t max_size = max_pages * WASM_PAGE_SIZE;
+    return max_size;
+  }
+
+  /* Reserve 4GiB. */
+  const uint64_t max_size = 0x100000000ul;
+  return max_size;
+#endif
+}
+
+#endif
+
+// Include operations for memory
+#define WASM_RT_MEM_OPS
+#include "wasm-rt-mem-impl-helper.inc"
+#undef WASM_RT_MEM_OPS
+
+// Include operations for shared memory
+#define WASM_RT_MEM_OPS_SHARED
+#include "wasm-rt-mem-impl-helper.inc"
+#undef WASM_RT_MEM_OPS_SHARED
+
+#undef C11_MEMORY_LOCK_VAR_INIT
+#undef C11_MEMORY_LOCK_AQUIRE
+#undef C11_MEMORY_LOCK_RELEASE
+#undef PTHREAD_MEMORY_LOCK_VAR_INIT
+#undef PTHREAD_MEMORY_LOCK_AQUIRE
+#undef PTHREAD_MEMORY_LOCK_RELEASE
+#undef WIN_MEMORY_LOCK_VAR_INIT
+#undef WIN_MEMORY_LOCK_AQUIRE
+#undef WIN_MEMORY_LOCK_RELEASE
+#undef WASM_PAGE_SIZE

third_party/wasm2c/wasm2c/wasm-rt.h

@@ -28,7 +28,7 @@ extern "C" {
 #endif
 
 #ifndef __has_builtin
-#define __has_builtin(x) 0  // Compatibility with non-clang compilers.
+#define __has_builtin(x) 0 /** Compatibility with non-clang compilers. */
 #endif
 
 #if __has_builtin(__builtin_expect)
@@ -51,14 +51,53 @@ extern "C" {
 #define wasm_rt_unreachable abort
 #endif
 
-#ifdef _MSC_VER
+#ifdef __STDC_VERSION__
-#define WASM_RT_THREAD_LOCAL __declspec(thread)
+#if __STDC_VERSION__ >= 201112L
-#elif defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)
+#define WASM_RT_C11_AVAILABLE
+#endif
+#endif
+
+/**
+ * Many devices don't implement the C11 threads.h. We use CriticalSection APIs
+ * for Windows and pthreads on other platforms where threads are not available.
+ */
+#ifdef WASM_RT_C11_AVAILABLE
+
+#if defined(_WIN32)
+#include <windows.h>
+#define WASM_RT_MUTEX CRITICAL_SECTION
+#define WASM_RT_USE_CRITICALSECTION 1
+#elif defined(__APPLE__) || defined(__STDC_NO_THREADS__)
+#include <pthread.h>
+#define WASM_RT_MUTEX pthread_mutex_t
+#define WASM_RT_USE_PTHREADS 1
+#else
+#include <threads.h>
+#define WASM_RT_MUTEX mtx_t
+#define WASM_RT_USE_C11THREADS 1
+#endif
+
+#endif
+
+#ifdef WASM_RT_C11_AVAILABLE
 #define WASM_RT_THREAD_LOCAL _Thread_local
+#elif defined(_MSC_VER)
+#define WASM_RT_THREAD_LOCAL __declspec(thread)
+#elif (defined(__GNUC__) || defined(__clang__)) && !defined(__APPLE__)
+// Disabled on Apple systems due to sporadic test failures.
+#define WASM_RT_THREAD_LOCAL __thread
 #else
 #define WASM_RT_THREAD_LOCAL
 #endif
 
+/**
+ * If enabled, perform additional sanity checks in the generated wasm2c code and
+ * wasm2c runtime. This is useful to enable on debug builds.
+ */
+#ifndef WASM_RT_SANITY_CHECKS
+#define WASM_RT_SANITY_CHECKS 0
+#endif
+
 /**
  * Backward compatibility: Convert the previously exposed
  * WASM_RT_MEMCHECK_SIGNAL_HANDLER macro to the ALLOCATION and CHECK macros that
@@ -107,17 +146,19 @@ extern "C" {
  *
  * This defaults to GUARD_PAGES as this is the fasest option, iff the
  * requirements of GUARD_PAGES --- 64-bit platforms, MMAP allocation strategy,
- * no 64-bit memories --- are met. This falls back to BOUNDS otherwise.
+ * no 64-bit memories, no big-endian --- are met. This falls back to BOUNDS
+ * otherwise.
  */
 
-// Check if Guard checks are supported
+/** Check if Guard checks are supported */
-#if UINTPTR_MAX > 0xffffffff && WASM_RT_USE_MMAP && !SUPPORT_MEMORY64
+#if UINTPTR_MAX > 0xffffffff && WASM_RT_USE_MMAP && !SUPPORT_MEMORY64 && \
+    !WABT_BIG_ENDIAN
 #define WASM_RT_GUARD_PAGES_SUPPORTED 1
 #else
 #define WASM_RT_GUARD_PAGES_SUPPORTED 0
 #endif
 
-// Specify defaults for memory checks if unspecified
+/** Specify defaults for memory checks if unspecified */
 #if !defined(WASM_RT_MEMCHECK_GUARD_PAGES) && \
     !defined(WASM_RT_MEMCHECK_BOUNDS_CHECK)
 #if WASM_RT_GUARD_PAGES_SUPPORTED
@@ -127,7 +168,7 @@ extern "C" {
 #endif
 #endif
 
-// Ensure the macros are defined
+/** Ensure the macros are defined */
 #ifndef WASM_RT_MEMCHECK_GUARD_PAGES
 #define WASM_RT_MEMCHECK_GUARD_PAGES 0
 #endif
@@ -135,7 +176,7 @@ extern "C" {
 #define WASM_RT_MEMCHECK_BOUNDS_CHECK 0
 #endif
 
-// Sanity check the use of guard pages
+/** Sanity check the use of guard pages */
 #if WASM_RT_MEMCHECK_GUARD_PAGES && !WASM_RT_GUARD_PAGES_SUPPORTED
 #error \
     "WASM_RT_MEMCHECK_GUARD_PAGES not supported on this platform/configuration"
@@ -166,17 +207,116 @@ extern "C" {
 #define WASM_RT_INSTALL_SIGNAL_HANDLER 0
 #endif
 
-#ifndef WASM_RT_USE_STACK_DEPTH_COUNT
+/**
-/* The signal handler on POSIX can detect call stack overflows. On windows, or
+ * This macro, if defined to 1 (i.e., allows the "segue" optimization), allows
- * platforms without a signal handler, we use stack depth counting. */
+ * Wasm2c to use segment registers to speedup access to the linear heap. Note
-#if WASM_RT_INSTALL_SIGNAL_HANDLER && !defined(_WIN32)
+ * that even if allowed in this way, the segment registers would only be used if
-#define WASM_RT_USE_STACK_DEPTH_COUNT 0
+ * Wasm2c output is compiled for a suitable architecture and OS and the produces
+ * C file is compiled by supported compilers. The extact restrictions are listed
+ * in detail in src/template/wasm2c.declarations.c
+ */
+#ifndef WASM_RT_ALLOW_SEGUE
+#define WASM_RT_ALLOW_SEGUE 0
+#endif
+
+/**
+ * The segue optimization restores x86 segments to their old values when exiting
+ * wasm2c code. If WASM_RT_SEGUE_FREE_SEGMENT is defined, wasm2c assumes it has
+ * exclusive use of the segment and optimizes performance in two ways. First, it
+ * does not restore the "old" value of the segment during exits. Second, wasm2c
+ * only sets the segment register if the value has changed since the last time
+ * it was set.
+ */
+#ifndef WASM_RT_SEGUE_FREE_SEGMENT
+#define WASM_RT_SEGUE_FREE_SEGMENT 0
+#endif
+
+#ifndef WASM_RT_USE_SEGUE
+// Memory functions can use the segue optimization if allowed. The segue
+// optimization uses x86 segments to point to a linear memory. We use this
+// optimization when:
+//
+// (1) Segue is allowed using WASM_RT_ALLOW_SEGUE
+// (2) on x86_64 without WABT_BIG_ENDIAN enabled
+// (3) the compiler supports: intrinsics for (rd|wr)gsbase, "address namespaces"
+//     for accessing pointers, and supports memcpy on pointers with custom
+//     "address namespaces". GCC does not support the memcpy requirement, so
+//     this leaves only clang (version 9 or later) for now.
+// (4) The OS provides a way to query if (rd|wr)gsbase is allowed by the kernel
+// or the implementation has to use a syscall for this.
+// (5) The OS doesn't replace the segment register on context switch which
+//     eliminates windows for now
+//
+// While more OS can be supported in the future, we only support linux for now
+#if WASM_RT_ALLOW_SEGUE && !WABT_BIG_ENDIAN &&                         \
+    (defined(__x86_64__) || defined(_M_X64)) && __clang__ &&           \
+    (__clang_major__ >= 9) && __has_builtin(__builtin_ia32_wrgsbase64) && \
+    !defined(_WIN32) && defined(__linux__)
+#define WASM_RT_USE_SEGUE 1
 #else
-#define WASM_RT_USE_STACK_DEPTH_COUNT 1
+#define WASM_RT_USE_SEGUE 0
 #endif
 #endif
 
-#if WASM_RT_USE_STACK_DEPTH_COUNT
+/**
+ * This macro, if defined, allows the embedder to disable all stack exhaustion
+ * checks. This a non conformant configuration, i.e., this does not respect
+ * Wasm's specification, and may compromise security. Use with caution.
+ */
+#ifndef WASM_RT_NONCONFORMING_UNCHECKED_STACK_EXHAUSTION
+#define WASM_RT_NONCONFORMING_UNCHECKED_STACK_EXHAUSTION 0
+#endif
+
+/**
+ * We need to detect and trap stack overflows. If we use a signal handler on
+ * POSIX systems, this can detect call stack overflows. On windows, or platforms
+ * without a signal handler, we use stack depth counting.
+ */
+#if !defined(WASM_RT_STACK_DEPTH_COUNT) &&        \
+    !defined(WASM_RT_STACK_EXHAUSTION_HANDLER) && \
+    !WASM_RT_NONCONFORMING_UNCHECKED_STACK_EXHAUSTION
+
+#if WASM_RT_INSTALL_SIGNAL_HANDLER && !defined(_WIN32)
+#define WASM_RT_STACK_EXHAUSTION_HANDLER 1
+#else
+#define WASM_RT_STACK_DEPTH_COUNT 1
+#endif
+
+#endif
+
+/** Ensure the stack macros are defined */
+#ifndef WASM_RT_STACK_DEPTH_COUNT
+#define WASM_RT_STACK_DEPTH_COUNT 0
+#endif
+#ifndef WASM_RT_STACK_EXHAUSTION_HANDLER
+#define WASM_RT_STACK_EXHAUSTION_HANDLER 0
+#endif
+
+#if WASM_RT_NONCONFORMING_UNCHECKED_STACK_EXHAUSTION
+
+#if (WASM_RT_STACK_EXHAUSTION_HANDLER + WASM_RT_STACK_DEPTH_COUNT) != 0
+#error \
+    "Cannot specify WASM_RT_NONCONFORMING_UNCHECKED_STACK_EXHAUSTION along with WASM_RT_STACK_EXHAUSTION_HANDLER or WASM_RT_STACK_DEPTH_COUNT"
+#endif
+
+#else
+
+#if (WASM_RT_STACK_EXHAUSTION_HANDLER + WASM_RT_STACK_DEPTH_COUNT) > 1
+#error \
+    "Cannot specify multiple options from WASM_RT_STACK_EXHAUSTION_HANDLER , WASM_RT_STACK_DEPTH_COUNT"
+#elif (WASM_RT_STACK_EXHAUSTION_HANDLER + WASM_RT_STACK_DEPTH_COUNT) == 0
+#error \
+    "Must specify one of WASM_RT_STACK_EXHAUSTION_HANDLER , WASM_RT_STACK_DEPTH_COUNT"
+#endif
+
+#endif
+
+#if WASM_RT_STACK_EXHAUSTION_HANDLER && !WASM_RT_INSTALL_SIGNAL_HANDLER
+#error \
+    "WASM_RT_STACK_EXHAUSTION_HANDLER  can only be used if WASM_RT_INSTALL_SIGNAL_HANDLER is enabled"
+#endif
+
+#if WASM_RT_STACK_DEPTH_COUNT
 /**
  * When the signal handler cannot be used to detect stack overflows, stack depth
  * is limited explicitly. The maximum stack depth before trapping can be
@@ -196,13 +336,42 @@ extern WASM_RT_THREAD_LOCAL uint32_t wasm_rt_call_stack_depth;
 
 #endif
 
+#if WASM_RT_USE_SEGUE
+/**
+ * The segue optimization uses x86 segments to point to a linear memory. If
+ * used, the runtime must query whether it can use the fast userspace wrgsbase
+ * instructions or whether it must invoke syscalls to set the segment base,
+ * depending on the supported CPU features. The result of this query is saved in
+ * this variable.
+ */
+extern bool wasm_rt_fsgsbase_inst_supported;
+/**
+ * If fast userspace wrgsbase instructions don't exist, the runtime most provide
+ * a function that invokes the OS' underlying syscall to set the segment base.
+ */
+void wasm_rt_syscall_set_segue_base(void* base);
+/**
+ * If fast userspace rdgsbase instructions don't exist, the runtime most provide
+ * a function that invokes the OS' underlying syscall to get the segment base.
+ */
+void* wasm_rt_syscall_get_segue_base();
+/**
+ * If WASM_RT_SEGUE_FREE_SEGMENT is defined, we must only set the segment
+ * register if it was changed since the last time it was set. The last value set
+ * on the segment register is stored in this variable.
+ */
+#if WASM_RT_SEGUE_FREE_SEGMENT
+extern WASM_RT_THREAD_LOCAL void* wasm_rt_last_segment_val;
+#endif
+#endif
+
 #if defined(_MSC_VER)
 #define WASM_RT_NO_RETURN __declspec(noreturn)
 #else
 #define WASM_RT_NO_RETURN __attribute__((noreturn))
 #endif
 
-#if defined(__APPLE__) && WASM_RT_INSTALL_SIGNAL_HANDLER
+#if defined(__APPLE__) && WASM_RT_STACK_EXHAUSTION_HANDLER
 #define WASM_RT_MERGED_OOB_AND_EXHAUSTION_TRAPS 1
 #else
 #define WASM_RT_MERGED_OOB_AND_EXHAUSTION_TRAPS 0
@@ -217,7 +386,8 @@ typedef enum {
   WASM_RT_TRAP_INVALID_CONVERSION, /** Conversion from NaN to integer. */
   WASM_RT_TRAP_UNREACHABLE,        /** Unreachable instruction executed. */
   WASM_RT_TRAP_CALL_INDIRECT,      /** Invalid call_indirect, for any reason. */
-  WASM_RT_TRAP_UNCAUGHT_EXCEPTION, /* Exception thrown and not caught */
+  WASM_RT_TRAP_UNCAUGHT_EXCEPTION, /** Exception thrown and not caught. */
+  WASM_RT_TRAP_UNALIGNED,          /** Unaligned atomic instruction executed. */
 #if WASM_RT_MERGED_OOB_AND_EXHAUSTION_TRAPS
   WASM_RT_TRAP_EXHAUSTION = WASM_RT_TRAP_OOB,
 #else
@@ -244,54 +414,111 @@ typedef enum {
  */
 typedef void (*wasm_rt_function_ptr_t)(void);
 
+/**
+ * A pointer to a "tail-callee" function, called by a tail-call
+ * trampoline or by another tail-callee function. (The definition uses a
+ * single-member struct to allow a recursive definition.)
+ */
+typedef struct wasm_rt_tailcallee_t {
+  void (*fn)(void** instance_ptr,
+             void* tail_call_stack,
+             struct wasm_rt_tailcallee_t* next);
+} wasm_rt_tailcallee_t;
+
 /**
  * The type of a function (an arbitrary number of param and result types).
  * This is represented as an opaque 256-bit ID.
  */
 typedef const char* wasm_rt_func_type_t;
 
-/** A function instance (the runtime representation of a function).
+/**
- * These can be stored in tables of type funcref, or used as values. */
+ * A function instance (the runtime representation of a function).
+ * These can be stored in tables of type funcref, or used as values.
+ */
 typedef struct {
   /** The function's type. */
   wasm_rt_func_type_t func_type;
-  /** The function. The embedder must know the actual C signature of the
+  /**
-   * function and cast to it before calling. */
+   * The function. The embedder must know the actual C signature of the function
+   * and cast to it before calling.
+   */
   wasm_rt_function_ptr_t func;
-  /** A function instance is a closure of the function over an instance
+  /** An alternate version of the function to be used when tail-called. */
+  wasm_rt_tailcallee_t func_tailcallee;
+  /**
+   * A function instance is a closure of the function over an instance
    * of the originating module. The module_instance element will be passed into
-   * the function at runtime. */
+   * the function at runtime.
+   */
   void* module_instance;
 } wasm_rt_funcref_t;
 
 /** Default (null) value of a funcref */
-static const wasm_rt_funcref_t wasm_rt_funcref_null_value = {NULL, NULL, NULL};
+#define wasm_rt_funcref_null_value \
+  ((wasm_rt_funcref_t){NULL, NULL, {NULL}, NULL})
 
 /** The type of an external reference (opaque to WebAssembly). */
 typedef void* wasm_rt_externref_t;
 
 /** Default (null) value of an externref */
-static const wasm_rt_externref_t wasm_rt_externref_null_value = NULL;
+#define wasm_rt_externref_null_value ((wasm_rt_externref_t){NULL})
 
 /** A Memory object. */
 typedef struct {
   /** The linear memory data, with a byte length of `size`. */
   uint8_t* data;
-  /** The current and maximum page count for this Memory object. If there is no
+  /** The current page count for this Memory object. */
-   * maximum, `max_pages` is 0xffffffffu (i.e. UINT32_MAX). */
+  uint64_t pages;
-  uint64_t pages, max_pages;
+  /**
+   * The maximum page count for this Memory object. If there is no maximum,
+   * `max_pages` is 0xffffffffu (i.e. UINT32_MAX).
+   */
+  uint64_t max_pages;
   /** The current size of the linear memory, in bytes. */
   uint64_t size;
   /** Is this memory indexed by u64 (as opposed to default u32) */
   bool is64;
 } wasm_rt_memory_t;
 
+#ifdef WASM_RT_C11_AVAILABLE
+/** A shared Memory object. */
+typedef struct {
+  /**
+   * The linear memory data, with a byte length of `size`. The memory is marked
+   * atomic as it is shared and may have to be accessed with different memory
+   * orders --- sequential when being accessed atomically, relaxed otherwise.
+   * Unfortunately, the C standard does not state what happens if there are
+   * overlaps in two memory accesses which have a memory order, e.g., an
+   * atomic32 being read from the same location an atomic64 is read. One way to
+   * prevent optimizations from assuming non-overlapping behavior as typically
+   * done in C is to mark the memory as volatile. Thus the memory is atomic and
+   * volatile.
+   */
+  _Atomic volatile uint8_t* data;
+  /** The current page count for this Memory object. */
+  uint64_t pages;
+  /**
+   * The maximum page count for this Memory object. If there is no maximum,
+   * `max_pages` is 0xffffffffu (i.e. UINT32_MAX).
+   */
+  uint64_t max_pages;
+  /** The current size of the linear memory, in bytes. */
+  uint64_t size;
+  /** Is this memory indexed by u64 (as opposed to default u32) */
+  bool is64;
+  /** Lock used to ensure operations such as memory grow are threadsafe */
+  WASM_RT_MUTEX mem_lock;
+} wasm_rt_shared_memory_t;
+#endif
+
 /** A Table of type funcref. */
 typedef struct {
   /** The table element data, with an element count of `size`. */
   wasm_rt_funcref_t* data;
-  /** The maximum element count of this Table object. If there is no maximum,
+  /**
-   * `max_size` is 0xffffffffu (i.e. UINT32_MAX). */
+   * The maximum element count of this Table object. If there is no maximum,
+   * `max_size` is 0xffffffffu (i.e. UINT32_MAX).
+   */
   uint32_t max_size;
   /** The current element count of the table. */
   uint32_t size;
@@ -301,8 +528,10 @@ typedef struct {
 typedef struct {
   /** The table element data, with an element count of `size`. */
   wasm_rt_externref_t* data;
-  /** The maximum element count of this Table object. If there is no maximum,
+  /**
-   * `max_size` is 0xffffffffu (i.e. UINT32_MAX). */
+   * The maximum element count of this Table object. If there is no maximum,
+   * `max_size` is 0xffffffffu (i.e. UINT32_MAX).
+   */
   uint32_t max_size;
   /** The current element count of the table. */
   uint32_t size;
@@ -317,17 +546,28 @@ bool wasm_rt_is_initialized(void);
 /** Free the runtime's state. */
 void wasm_rt_free(void);
 
-/**
+/*
- * A hardened jmp_buf that allows checking for initialization before use
+ * Initialize the multithreaded runtime for a given thread. Must be
+ * called by each thread (other than the one that called wasm_rt_init)
+ * before initializing a Wasm module or calling an exported
+ * function.
  */
+void wasm_rt_init_thread(void);
+
+/*
+ * Free the individual thread's state.
+ */
+void wasm_rt_free_thread(void);
+
+/** A hardened jmp_buf that allows checking for initialization before use */
 typedef struct {
-  /* Is the jmp buf intialized? */
+  /** Is the jmp buf intialized? */
   bool initialized;
-  /* jmp_buf contents */
+  /** jmp_buf contents */
   jmp_buf buffer;
 } wasm_rt_jmp_buf;
 
-#if WASM_RT_INSTALL_SIGNAL_HANDLER && !defined(_WIN32)
+#ifndef _WIN32
 #define WASM_RT_SETJMP_SETBUF(buf) sigsetjmp(buf, 1)
 #else
 #define WASM_RT_SETJMP_SETBUF(buf) setjmp(buf)
@@ -336,17 +576,17 @@ typedef struct {
 #define WASM_RT_SETJMP(buf) \
   ((buf).initialized = true, WASM_RT_SETJMP_SETBUF((buf).buffer))
 
-#if WASM_RT_INSTALL_SIGNAL_HANDLER && !defined(_WIN32)
+#ifndef _WIN32
 #define WASM_RT_LONGJMP_UNCHECKED(buf, val) siglongjmp(buf, val)
 #else
 #define WASM_RT_LONGJMP_UNCHECKED(buf, val) longjmp(buf, val)
 #endif
 
-#define WASM_RT_LONGJMP(buf, val)                                  \
+#define WASM_RT_LONGJMP(buf, val)                                   \
-  /* Abort on failure as this may be called in the trap handler */ \
+  /** Abort on failure as this may be called in the trap handler */ \
-  if (!((buf).initialized))                                        \
+  if (!((buf).initialized))                                         \
-    abort();                                                       \
+    abort();                                                        \
-  (buf).initialized = false;                                       \
+  (buf).initialized = false;                                        \
   WASM_RT_LONGJMP_UNCHECKED((buf).buffer, val)
 
 /**
@@ -357,9 +597,7 @@ typedef struct {
  */
 WASM_RT_NO_RETURN void wasm_rt_trap(wasm_rt_trap_t);
 
-/**
+/** Return a human readable error string based on a trap type. */
- * Return a human readable error string based on a trap type.
- */
 const char* wasm_rt_strerror(wasm_rt_trap_t trap);
 
 #define wasm_rt_try(target) WASM_RT_SETJMP(target)
@@ -397,11 +635,23 @@ void wasm_rt_allocate_memory(wasm_rt_memory_t*,
  */
 uint64_t wasm_rt_grow_memory(wasm_rt_memory_t*, uint64_t pages);
 
-/**
+/** Free a Memory object. */
- * Free a Memory object.
- */
 void wasm_rt_free_memory(wasm_rt_memory_t*);
 
+#ifdef WASM_RT_C11_AVAILABLE
+/** Shared memory version of wasm_rt_allocate_memory */
+void wasm_rt_allocate_memory_shared(wasm_rt_shared_memory_t*,
+                                    uint64_t initial_pages,
+                                    uint64_t max_pages,
+                                    bool is64);
+
+/** Shared memory version of wasm_rt_grow_memory */
+uint64_t wasm_rt_grow_memory_shared(wasm_rt_shared_memory_t*, uint64_t pages);
+
+/** Shared memory version of wasm_rt_free_memory */
+void wasm_rt_free_memory_shared(wasm_rt_shared_memory_t*);
+#endif
+
 /**
  * Initialize a funcref Table object with an element count of `elements` and a
  * maximum size of `max_elements`.
@@ -416,9 +666,7 @@ void wasm_rt_allocate_funcref_table(wasm_rt_funcref_table_t*,
                                     uint32_t elements,
                                     uint32_t max_elements);
 
-/**
+/** Free a funcref Table object. */
- * Free a funcref Table object.
- */
 void wasm_rt_free_funcref_table(wasm_rt_funcref_table_t*);
 
 /**
@@ -430,9 +678,7 @@ void wasm_rt_allocate_externref_table(wasm_rt_externref_table_t*,
                                       uint32_t elements,
                                       uint32_t max_elements);
 
-/**
+/** Free an externref Table object. */
- * Free an externref Table object.
- */
 void wasm_rt_free_externref_table(wasm_rt_externref_table_t*);
 
 /**