Rename era pack to e2store & add support for E2HS file format (#3954)

* reorganize package into directory per format

* redefine Version and BlockIndex as CommonTypes

* define e2hs types

* move blockIndex functions to shared blockIndex.ts file

* write createBlockIndex function and replace code in era1

* implement e2hs format

* rename parse to decompress

* write parse function

* fix up blockTuple functions

* update blockTuple calls in e2hs code

* log better error on type mismatch

* test using reference file

* fix imports in tests

* rename era package to e2store

---------

Co-authored-by: Holger Drewes <[email protected]>

Author: ScottyPoi

Dockerfile

@@ -13,7 +13,7 @@ COPY packages/blockchain/dist packages/blockchain/dist
 COPY packages/client/dist packages/client/dist
 COPY packages/common/dist packages/common/dist
 COPY packages/devp2p/dist packages/devp2p/dist
-COPY packages/era/dist packages/era/dist
+COPY packages/e2store/dist packages/e2store/dist
 COPY packages/ethash/dist packages/ethash/dist
 COPY packages/evm/dist packages/evm/dist
 COPY packages/genesis/dist packages/genesis/dist
@@ -33,7 +33,7 @@ COPY packages/blockchain/package.json packages/blockchain/package.json
 COPY packages/client/package.json packages/client/package.json
 COPY packages/common/package.json packages/common/package.json
 COPY packages/devp2p/package.json packages/devp2p/package.json
-COPY packages/era/package.json packages/era/package.json
+COPY packages/e2store/package.json packages/e2store/package.json
 COPY packages/ethash/package.json packages/ethash/package.json
 COPY packages/evm/package.json packages/evm/package.json
 COPY packages/genesis/package.json packages/genesis/package.json

package-lock.json

@@ -1,43 +1,48 @@
-# @ethereumjs/era `v10`
+# @ethereumjs/e2store `v10`
 
-[![NPM Package][era-npm-badge]][era-npm-link]
-[![GitHub Issues][era-issues-badge]][era-issues-link]
-[![Actions Status][era-actions-badge]][era-actions-link]
-[![Code Coverage][era-coverage-badge]][era-coverage-link]
+[![NPM Package][e2store-npm-badge]][e2store-npm-link]
+[![GitHub Issues][e2store-issues-badge]][e2store-issues-link]
+[![Actions Status][e2store-actions-badge]][e2store-actions-link]
+[![Code Coverage][e2store-coverage-badge]][e2store-coverage-link]
 [![Discord][discord-badge]][discord-link]
 
 | A collection of utility functions for Ethereum. |
 | ----------------------------------------------- |
 
 ## Table of Contents
 
-- [Installation](#installation)
-- [Usage](#usage)
-- [EthereumJS](#ethereumjs)
-- [License](#license)
+- [@ethereumjs/e2store `v10`](#ethereumjse2store-v10)
+  - [Table of Contents](#table-of-contents)
+  - [Installation](#installation)
+  - [Usage](#usage)
+    - [Export History as Era1](#export-history-as-era1)
+    - [Read Era1 file](#read-era1-file)
+    - [Read Era file](#read-era-file)
+  - [EthereumJS](#ethereumjs)
+  - [License](#license)
 
 ## Installation
 
 To obtain the latest version, simply require the project using `npm`:
 
 ```shell
-npm install @ethereumjs/era
+npm install @ethereumjs/e2store
 ```
 
 ## Usage
 
 All helpers are re-exported from the root level and deep imports are not necessary. So an import can be done like this:
 
 ```ts
-import { formatEntry } from "@ethereumjs/era"
+import { formatEntry } from "@ethereumjs/e2store"
 ```
 
 ### Export History as Era1
 
 Export history in epochs of 8192 blocks as Era1 files
 
 ```ts
-import { exportEpochAsEra1 } from "@ethereumjs/era"
+import { exportEpochAsEra1 } from "@ethereumjs/e2store"
 
 const dataDir = PATH_TO_ETHEREUMJS_CLIENT_DB
 const epoch = 0
@@ -59,7 +64,7 @@ import {
   blockFromTuple,
   getHeaderRecords,
   EpochAccumulator,
-} from "@ethereumjs/era"
+} from "@ethereumjs/e2store"
 
 const era1File = readBinaryFile(PATH_TO_ERA1_FILE)
 
@@ -84,7 +89,7 @@ const epochAccumulatorRoot = EpochAccumulator.merkleRoot(headerRecords)
 ### Read Era file
 
 ```ts
-import { readBeaconState } from "@ethereumjs/era"
+import { readBeaconState } from "@ethereumjs/e2store"
 
 const eraFile = readBinaryFile(PATH_TO_ERA_FILE)
 
@@ -109,13 +114,13 @@ See our organizational [documentation](https://ethereumjs.readthedocs.io) for an
 
 [MPL-2.0](<https://tldrlegal.com/license/mozilla-public-license-2.0-(mpl-2)>)
 
-[era-npm-badge]: https://img.shields.io/npm/v/@ethereumjs/era.svg
-[era-npm-link]: https://www.npmjs.org/package/@ethereumjs/era
-[era-issues-badge]: https://img.shields.io/github/issues/ethereumjs/ethereumjs-monorepo/package:%20era?label=issues
-[era-issues-link]: https://github.com/ethereumjs/ethereumjs-monorepo/issues?q=is%3Aopen+is%3Aissue+label%3A"package%3A+era"
-[era-actions-badge]: https://github.com/ethereumjs/ethereumjs-monorepo/workflows/Era/badge.svg
-[era-actions-link]: https://github.com/ethereumjs/ethereumjs-monorepo/actions?query=workflow%3A%22Era%22
-[era-coverage-badge]: https://codecov.io/gh/ethereumjs/ethereumjs-monorepo/branch/master/graph/badge.svg?flag=era
-[era-coverage-link]: https://codecov.io/gh/ethereumjs/ethereumjs-monorepo/tree/master/packages/era
+[e2store-npm-badge]: https://img.shields.io/npm/v/@ethereumjs/e2store.svg
+[e2store-npm-link]: https://www.npmjs.org/package/@ethereumjs/e2store
+[e2store-issues-badge]: https://img.shields.io/github/issues/ethereumjs/ethereumjs-monorepo/package:%20e2store?label=issues
+[e2store-issues-link]: https://github.com/ethereumjs/ethereumjs-monorepo/issues?q=is%3Aopen+is%3Aissue+label%3A"package%3A+e2store"
+[e2store-actions-badge]: https://github.com/ethereumjs/ethereumjs-monorepo/workflows/E2Store/badge.svg
+[e2store-actions-link]: https://github.com/ethereumjs/ethereumjs-monorepo/actions?query=workflow%3A%22E2Store%22
+[e2store-coverage-badge]: https://codecov.io/gh/ethereumjs/ethereumjs-monorepo/branch/master/graph/badge.svg?flag=e2store
+[e2store-coverage-link]: https://codecov.io/gh/ethereumjs/ethereumjs-monorepo/tree/master/packages/e2store
 [discord-badge]: https://img.shields.io/static/v1?logo=discord&label=discord&message=Join&color=blue
 [discord-link]: https://discord.gg/TNwARpR

packages/era/.c8rc.json packages/e2store/.c8rc.jsonpackages/era/.c8rc.json renamed to packages/e2store/.c8rc.json

@@ -1,11 +1,11 @@
 {
-  "name": "@ethereumjs/era",
+  "name": "@ethereumjs/e2store",
   "version": "10.0.0-dev-rc.1",
-  "description": "Era file support for EthereumJS",
-  "keywords": ["ethereum", "era", "era1"],
-  "homepage": "https://github.com/ethereumjs/ethereumjs-monorepo/tree/master/packages/era#readme",
+  "description": "E2Store support for EthereumJS",
+  "keywords": ["ethereum", "e2store", "e2hs", "era", "era1", "era2", "storage", "snappy"],
+  "homepage": "https://github.com/ethereumjs/ethereumjs-monorepo/tree/master/packages/e2store#readme",
   "bugs": {
-    "url": "https://github.com/ethereumjs/ethereumjs-monorepo/issues?q=is%3Aissue+label%3A%22package%3A+era%22"
+    "url": "https://github.com/ethereumjs/ethereumjs-monorepo/issues?q=is%3Aissue+label%3A%22package%3A+e2store%22"
   },
   "repository": {
     "type": "git",

packages/era/.gitignore packages/e2store/.gitignorepackages/era/.gitignore renamed to packages/e2store/.gitignore

@@ -0,0 +1,57 @@
+import { EthereumJSErrorWithoutCode } from '@ethereumjs/rlp'
+import { bigInt64ToBytes, bytesToBigInt64, concatBytes, equalsBytes } from '@ethereumjs/util'
+import { formatEntry, readEntry } from './e2store.ts'
+import { CommonTypes, VERSION } from './types.ts'
+
+export function getBlockIndex(bytes: Uint8Array) {
+  const count = Number(bytesToBigInt64(bytes.slice(-8), true))
+  const recordLength = 8 * count + 24
+  const recordEnd = bytes.length
+  const recordStart = recordEnd - recordLength
+  const { data, type } = readEntry(bytes.subarray(recordStart, recordEnd))
+  if (!equalsBytes(type, CommonTypes.BlockIndex)) {
+    throw EthereumJSErrorWithoutCode(
+      `Expected block index (type: ${CommonTypes.BlockIndex}) but got ${type}`,
+    )
+  }
+  return { data, type, count, recordStart }
+}
+
+export function readBlockIndex(data: Uint8Array, count: number) {
+  const startingNumber = Number(bytesToBigInt64(data.slice(0, 8), true))
+  const offsets: number[] = []
+  for (let i = 0; i < count; i++) {
+    const slotEntry = data.subarray((i + 1) * 8, (i + 2) * 8)
+    const offset = Number(new DataView(slotEntry.slice(0, 8).buffer).getBigInt64(0, true))
+    offsets.push(offset)
+  }
+  return {
+    startingNumber,
+    offsets,
+  }
+}
+
+export async function createBlockIndex(blockTuples: Uint8Array[], startingNumber: bigint) {
+  const version = await formatEntry(VERSION)
+  const tuplesLength = blockTuples.reduce((acc, b) => acc + b.length, 0)
+  const count = bigInt64ToBytes(BigInt(blockTuples.length), true)
+  const blockIndexLength = 8 * blockTuples.length + 24
+  const e2hsLength = version.length + tuplesLength + count.length + blockIndexLength
+  const recordStart = e2hsLength - blockIndexLength
+  const offsetBigInt: bigint[] = []
+  for (let i = 0; i < blockTuples.length; i++) {
+    if (i === 0) {
+      const offset = 8 - recordStart
+      offsetBigInt.push(BigInt(offset))
+    } else {
+      const offset = offsetBigInt[i - 1] + BigInt(blockTuples[i - 1].length)
+      offsetBigInt.push(offset)
+    }
+  }
+  const offsets: Uint8Array[] = offsetBigInt.map((o) => bigInt64ToBytes(o, true))
+  const blockIndex = await formatEntry({
+    type: CommonTypes.BlockIndex,
+    data: concatBytes(bigInt64ToBytes(startingNumber, true), ...offsets, count),
+  })
+  return blockIndex
+}