feat: header/data and sequencer/block-builder separation

.golangci.yml

@@ -32,6 +32,9 @@ issues:
  - EXC0014 # EXC0014 revive: Annoying issue about not having a comment. The rare codebase has such comments
 
 linters-settings:
+ gosec:
+ excludes:
+ - G115
  revive:
  rules:
  - name: package-comments

block/batch.go

@@ -0,0 +1,37 @@
+package block
+
+import (
+	"sync"
+)
+
+// BatchQueue is a queue of transaction batches with timestamps
+type BatchQueue struct {
+	queue []BatchWithTime
+	mu sync.Mutex
+}
+
+// NewBatchQueue creates a new BatchQueue
+func NewBatchQueue() *BatchQueue {
+	return &BatchQueue{
+ queue: make([]BatchWithTime, 0),
+	}
+}
+
+// AddBatch adds a new batch to the queue
+func (bq *BatchQueue) AddBatch(batch BatchWithTime) {
+	bq.mu.Lock()
+	defer bq.mu.Unlock()
+	bq.queue = append(bq.queue, batch)
+}
+
+// Next returns the next batch in the queue
+func (bq *BatchQueue) Next() *BatchWithTime {
+	bq.mu.Lock()
+	defer bq.mu.Unlock()
+	if len(bq.queue) == 0 {
+ return nil
+	}
+	batch := bq.queue[0]
+	bq.queue = bq.queue[1:]
+	return &batch
+}

block/batch_test.go

@@ -0,0 +1,69 @@
+package block
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/rollkit/go-sequencing"
+)
+
+var (
+	batch1 = BatchWithTime{&sequencing.Batch{
+ Transactions: []sequencing.Tx{
+ sequencing.Tx("batch1"),
+ },
+	}, time.Time{}}
+	batch2 = BatchWithTime{&sequencing.Batch{
+ Transactions: []sequencing.Tx{
+ sequencing.Tx("batch2"),
+ },
+	}, time.Time{}}
+)
+
+func TestNewBatchQueue(t *testing.T) {
+	// Test creating a new BatchQueue
+	bq := NewBatchQueue()
+	require.NotNil(t, bq, "NewBatchQueue should return a non-nil instance")
+	require.Empty(t, bq.queue, "New BatchQueue should have an empty queue")
+}
+
+func TestBatchQueue_AddBatch(t *testing.T) {
+	// Create a new BatchQueue
+	bq := NewBatchQueue()
+
+	// Add the first batch and check
+	bq.AddBatch(batch1)
+	require.Len(t, bq.queue, 1, "BatchQueue should have 1 batch after adding")
+	require.Equal(t, batch1, bq.queue[0], "The first batch should match the one added")
+
+	// Add the second batch and check
+	bq.AddBatch(batch2)
+	require.Len(t, bq.queue, 2, "BatchQueue should have 2 batches after adding another")
+	require.Equal(t, batch2, bq.queue[1], "The second batch should match the one added")
+}
+
+func TestBatchQueue_Next(t *testing.T) {
+	// Create a new BatchQueue
+	bq := NewBatchQueue()
+
+	// Test with empty queue
+	require.Nil(t, bq.Next(), "Next should return nil when the queue is empty")
+
+	// Add batches
+	bq.AddBatch(batch1)
+	bq.AddBatch(batch2)
+
+	// Retrieve the first batch
+	nextBatch := bq.Next()
+	require.NotNil(t, nextBatch, "Next should return the first batch when called")
+	require.Equal(t, batch1, *nextBatch, "Next should return the first batch added")
+	require.Len(t, bq.queue, 1, "BatchQueue should have 1 batch after retrieving the first")
+
+	// Retrieve the second batch
+	nextBatch = bq.Next()
+	require.NotNil(t, nextBatch, "Next should return the second batch when called")
+	require.Equal(t, batch2, *nextBatch, "Next should return the second batch added")
+	require.Empty(t, bq.queue, "BatchQueue should be empty after retrieving all batches")
+}

block/block-manager.md

@@ -156,7 +156,7 @@ The communication between the full node and block manager:
 * The sequencer can produce empty blocks.
 * The block manager uses persistent storage (disk) when the `root_dir` and `db_path` configuration parameters are specified in `config.toml` file under the app directory. If these configuration parameters are not specified, the in-memory storage is used, which will not be persistent if the node stops.
 * The block manager does not re-apply the block again (in other words, create a new updated state and persist it) when a block was initially applied using P2P block sync, but later was DA included during DA retrieval. The block is only marked DA included in this case.
-* The block sync store is created by prefixing `blockSync` on the main data store.
+* The data sync store is created by prefixing `dataSync` on the main data store.
 * The genesis `ChainID` is used to create the `PubSubTopID` in go-header with the string `-block` appended to it. This append is because the full node also has a P2P header sync running with a different P2P network. Refer to go-header specs for more details.
 * Block sync over the P2P network works only when a full node is connected to the P2P network by specifying the initial seeds to connect to via `P2PConfig.Seeds` configuration parameter when starting the full node.
 * Node's context is passed down to all the components of the P2P block sync to control shutting down the service either abruptly (in case of failure) or gracefully (during successful scenarios).

block/cache_test.go

@@ -0,0 +1,66 @@
+package block
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/rollkit/rollkit/types"
+)
+
+func TestBlockCache(t *testing.T) {
+	require := require.New(t)
+	// Create new HeaderCache and DataCache and verify not nil
+	hc := NewHeaderCache()
+	require.NotNil(hc)
+
+	dc := NewDataCache()
+	require.NotNil(dc)
+
+	// Test setBlock and getBlock
+	height, nTxs := uint64(1), 2
+	header, data := types.GetRandomBlock(height, nTxs)
+	hc.setHeader(height, header)
+	gotHeader := hc.getHeader(height)
+	require.NotNil(gotHeader, "getHeader should return non-nil after setHeader")
+	require.Equal(header, gotHeader)
+	dc.setData(height, data)
+	gotData := dc.getData(height)
+	require.NotNil(gotData, "getData should return non-nil after setData")
+	require.Equal(data, gotData)
+
+	// Test overwriting a block
+	header1, data1 := types.GetRandomBlock(height, nTxs)
+	hc.setHeader(height, header1)
+	gotHeader1 := hc.getHeader(height)
+	require.NotNil(gotHeader1, "getHeader should return non-nil after overwriting a header")
+	require.Equal(header1, gotHeader1)
+	dc.setData(height, data1)
+	gotData1 := dc.getData(height)
+	require.NotNil(gotData1, "getData should return non-nil after overwriting a data")
+	require.Equal(data1, gotData1)
+
+	// Test deleteBlock
+	hc.deleteHeader(height)
+	h := hc.getHeader(height)
+	require.Nil(h, "getHeader should return nil after deleteHeader")
+	dc.deleteData(height)
+	d := dc.getData(height)
+	require.Nil(d, "getData should return nil after deleteData")
+
+	// Test isSeen and setSeen
+	require.False(hc.isSeen("hash"), "isSeen should return false for unseen hash")
+	hc.setSeen("hash")
+	require.True(hc.isSeen("hash"), "isSeen should return true for seen hash")
+	require.False(dc.isSeen("hash"), "isSeen should return false for unseen hash")
+	dc.setSeen("hash")
+	require.True(dc.isSeen("hash"), "isSeen should return true for seen hash")
+
+	// Test setDAIncluded
+	require.False(hc.isDAIncluded("hash"), "DAIncluded should be false for unseen hash")
+	hc.setDAIncluded("hash")
+	require.True(hc.isDAIncluded("hash"), "DAIncluded should be true for seen hash")
+	require.False(dc.isDAIncluded("hash"), "DAIncluded should be false for unseen hash")
+	dc.setDAIncluded("hash")
+	require.True(dc.isDAIncluded("hash"), "DAIncluded should be true for seen hash")
+}

block/data_cache.go

@@ -0,0 +1,63 @@
+package block
+
+import (
+	"sync"
+
+	"github.com/rollkit/rollkit/types"
+)
+
+// DataCache maintains data that are seen and hard confirmed
+type DataCache struct {
+	data *sync.Map
+	hashes *sync.Map
+	daIncluded *sync.Map
+}
+
+// NewDataCache returns a new DataCache struct
+func NewDataCache() *DataCache {
+	return &DataCache{
+ data: new(sync.Map),
+ hashes: new(sync.Map),
+ daIncluded: new(sync.Map),
+	}
+}
+
+func (hc *DataCache) getData(height uint64) *types.Data {
+	data, ok := hc.data.Load(height)
+	if !ok {
+ return nil
+	}
+	return data.(*types.Data)
+}
+
+func (hc *DataCache) setData(height uint64, data *types.Data) {
+	hc.data.Store(height, data)
+}
+
+func (hc *DataCache) deleteData(height uint64) {
+	hc.data.Delete(height)
+}
+
+func (hc *DataCache) isSeen(hash string) bool {
+	seen, ok := hc.hashes.Load(hash)
+	if !ok {
+ return false
+	}
+	return seen.(bool)
+}
+
+func (hc *DataCache) setSeen(hash string) {
+	hc.hashes.Store(hash, true)
+}
+
+func (hc *DataCache) isDAIncluded(hash string) bool {
+	daIncluded, ok := hc.daIncluded.Load(hash)
+	if !ok {
+ return false
+	}
+	return daIncluded.(bool)
+}
+
+func (hc *DataCache) setDAIncluded(hash string) {
+	hc.daIncluded.Store(hash, true)
+}