kernel
/
git
mirror of https://git.kernel.org/pub/scm/git/git.git
1
0
Fork 0
Code Releases Activity

Merge branch 'ps/reftable-api-revamp' 

Overhaul of the reftable API.

* ps/reftable-api-revamp:
  reftable/table: move printing logic into test helper
  reftable/constants: make block types part of the public interface
  reftable/table: introduce iterator for table blocks
  reftable/table: add `reftable_table` to the public interface
  reftable/block: expose a generic iterator over reftable records
  reftable/block: make block iterators reseekable
  reftable/block: store block pointer in the block iterator
  reftable/block: create public interface for reading blocks
  git-zlib: use `struct z_stream_s` instead of typedef
  reftable/block: rename `block_reader` to `reftable_block`
  reftable/block: rename `block` to `block_data`
  reftable/table: move reading block into block reader
  reftable/block: simplify how we track restart points
  reftable/blocksource: consolidate code into a single file
  reftable/reader: rename data structure to "table"
  reftable: fix formatting of the license header
main
Junio C Hamano 2025-04-29 14:21:29 -07:00
parent 0c9d6b7ced e0011188ca
commit a819a3da85
54 changed files with 1640 additions and 1279 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Documentation/howto/recover-corrupted-object-harder.adoc
View File

@ -125,7 +125,7 @@ static int try_zlib(unsigned char *buf, int len)
{
/* make this absurdly large so we don't have to loop */
static unsigned char out[1024*1024];
z_stream z;
struct z_stream_s z;
int ret;

memset(&z, 0, sizeof(z));
@ -278,7 +278,7 @@ int main(int argc, char **argv)
static unsigned char buf[25 * 1024 * 1024];
static unsigned char out[25 * 1024 * 1024];
int len;
z_stream z;
struct z_stream_s z;
int ret;

len = read(0, buf, sizeof(buf));

4
Makefile
View File

@ -1377,10 +1377,10 @@ UNIT_TEST_PROGRAMS += t-reftable-basics
UNIT_TEST_PROGRAMS += t-reftable-block
UNIT_TEST_PROGRAMS += t-reftable-merged
UNIT_TEST_PROGRAMS += t-reftable-pq
UNIT_TEST_PROGRAMS += t-reftable-reader
UNIT_TEST_PROGRAMS += t-reftable-readwrite
UNIT_TEST_PROGRAMS += t-reftable-record
UNIT_TEST_PROGRAMS += t-reftable-stack
UNIT_TEST_PROGRAMS += t-reftable-table
UNIT_TEST_PROGS = $(patsubst %,$(UNIT_TEST_BIN)/%$X,$(UNIT_TEST_PROGRAMS))
UNIT_TEST_OBJS += $(UNIT_TEST_DIR)/test-lib.o
UNIT_TEST_OBJS += $(UNIT_TEST_DIR)/lib-reftable.o
@ -2737,10 +2737,10 @@ REFTABLE_OBJS += reftable/blocksource.o
REFTABLE_OBJS += reftable/iter.o
REFTABLE_OBJS += reftable/merged.o
REFTABLE_OBJS += reftable/pq.o
REFTABLE_OBJS += reftable/reader.o
REFTABLE_OBJS += reftable/record.o
REFTABLE_OBJS += reftable/stack.o
REFTABLE_OBJS += reftable/system.o
REFTABLE_OBJS += reftable/table.o
REFTABLE_OBJS += reftable/tree.o
REFTABLE_OBJS += reftable/writer.o


4
compat/zlib-compat.h
View File

@ -4,8 +4,8 @@
#ifdef HAVE_ZLIB_NG
# include <zlib-ng.h>

# define z_stream zng_stream
#define gz_header_s zng_gz_header_s
# define z_stream_s zng_stream_s
# define gz_header_s zng_gz_header_s

# define crc32(crc, buf, len) zng_crc32(crc, buf, len)


2
git-zlib.h
View File

@ -4,7 +4,7 @@
#include "compat/zlib-compat.h"

typedef struct git_zstream {
z_stream z;
struct z_stream_s z;
unsigned long avail_in;
unsigned long avail_out;
unsigned long total_in;

2
meson.build
View File

@ -446,10 +446,10 @@ libgit_sources = [
'reftable/iter.c',
'reftable/merged.c',
'reftable/pq.c',
'reftable/reader.c',
'reftable/record.c',
'reftable/stack.c',
'reftable/system.c',
'reftable/table.c',
'reftable/tree.c',
'reftable/writer.c',
'remote.c',

12
reftable/basics.c
View File

@ -1,10 +1,10 @@
/*
Copyright 2020 Google LLC

Use of this source code is governed by a BSD-style
license that can be found in the LICENSE file or at
https://developers.google.com/open-source/licenses/bsd
*/
* Copyright 2020 Google LLC
*
* Use of this source code is governed by a BSD-style
* license that can be found in the LICENSE file or at
* https://developers.google.com/open-source/licenses/bsd
*/

#define REFTABLE_ALLOW_BANNED_ALLOCATORS
#include "basics.h"

19
reftable/basics.h
View File

@ -1,10 +1,10 @@
/*
Copyright 2020 Google LLC

Use of this source code is governed by a BSD-style
license that can be found in the LICENSE file or at
https://developers.google.com/open-source/licenses/bsd
*/
* Copyright 2020 Google LLC
*
* Use of this source code is governed by a BSD-style
* license that can be found in the LICENSE file or at
* https://developers.google.com/open-source/licenses/bsd
*/

#ifndef BASICS_H
#define BASICS_H
@ -18,13 +18,6 @@ https://developers.google.com/open-source/licenses/bsd

#define REFTABLE_UNUSED __attribute__((__unused__))

struct reftable_buf {
size_t alloc;
size_t len;
char *buf;
};
#define REFTABLE_BUF_INIT { 0 }

/*
* Initialize the buffer such that it is ready for use. This is equivalent to
* using REFTABLE_BUF_INIT for stack-allocated variables.

284
reftable/block.c
View File

@ -1,15 +1,16 @@
/*
Copyright 2020 Google LLC

Use of this source code is governed by a BSD-style
license that can be found in the LICENSE file or at
https://developers.google.com/open-source/licenses/bsd
*/
* Copyright 2020 Google LLC
*
* Use of this source code is governed by a BSD-style
* license that can be found in the LICENSE file or at
* https://developers.google.com/open-source/licenses/bsd
*/

#include "block.h"

#include "blocksource.h"
#include "constants.h"
#include "iter.h"
#include "record.h"
#include "reftable-error.h"
#include "system.h"
@ -160,7 +161,7 @@ int block_writer_finish(struct block_writer *w)
* Log records are stored zlib-compressed. Note that the compression
* also spans over the restart points we have just written.
*/
if (block_writer_type(w) == BLOCK_TYPE_LOG) {
if (block_writer_type(w) == REFTABLE_BLOCK_TYPE_LOG) {
int block_header_skip = 4 + w->header_off;
uLongf src_len = w->next - block_header_skip, compressed_len;
int ret;
@ -210,61 +211,86 @@ int block_writer_finish(struct block_writer *w)
return w->next;
}

int block_reader_init(struct block_reader *br, struct reftable_block *block,
uint32_t header_off, uint32_t table_block_size,
uint32_t hash_size)
static int read_block(struct reftable_block_source *source,
struct reftable_block_data *dest, uint64_t off,
uint32_t sz)
{
size_t size = block_source_size(source);
block_source_release_data(dest);
if (off >= size)
return 0;
if (off + sz > size)
sz = size - off;
return block_source_read_data(source, dest, off, sz);
}

int reftable_block_init(struct reftable_block *block,
struct reftable_block_source *source,
uint32_t offset, uint32_t header_size,
uint32_t table_block_size, uint32_t hash_size)
{
uint32_t guess_block_size = table_block_size ?
table_block_size : DEFAULT_BLOCK_SIZE;
uint32_t full_block_size = table_block_size;
uint8_t typ = block->data[header_off];
uint32_t sz = reftable_get_be24(block->data + header_off + 1);
int err = 0;
uint16_t restart_count = 0;
uint32_t restart_start = 0;
uint8_t *restart_bytes = NULL;
uint16_t restart_count;
uint32_t restart_off;
uint32_t block_size;
uint8_t block_type;
int err;

reftable_block_done(&br->block);
err = read_block(source, &block->block_data, offset, guess_block_size);
if (err < 0)
goto done;

if (!reftable_is_block_type(typ)) {
err = REFTABLE_FORMAT_ERROR;
block_type = block->block_data.data[header_size];
if (!reftable_is_block_type(block_type)) {
err = REFTABLE_FORMAT_ERROR;
goto done;
}

if (typ == BLOCK_TYPE_LOG) {
uint32_t block_header_skip = 4 + header_off;
uLong dst_len = sz - block_header_skip;
uLong src_len = block->len - block_header_skip;
block_size = reftable_get_be24(block->block_data.data + header_size + 1);
if (block_size > guess_block_size) {
err = read_block(source, &block->block_data, offset, block_size);
if (err < 0)
goto done;
}

if (block_type == REFTABLE_BLOCK_TYPE_LOG) {
uint32_t block_header_skip = 4 + header_size;
uLong dst_len = block_size - block_header_skip;
uLong src_len = block->block_data.len - block_header_skip;

/* Log blocks specify the *uncompressed* size in their header. */
REFTABLE_ALLOC_GROW_OR_NULL(br->uncompressed_data, sz,
br->uncompressed_cap);
if (!br->uncompressed_data) {
REFTABLE_ALLOC_GROW_OR_NULL(block->uncompressed_data, block_size,
block->uncompressed_cap);
if (!block->uncompressed_data) {
err = REFTABLE_OUT_OF_MEMORY_ERROR;
goto done;
}

/* Copy over the block header verbatim. It's not compressed. */
memcpy(br->uncompressed_data, block->data, block_header_skip);
memcpy(block->uncompressed_data, block->block_data.data, block_header_skip);

if (!br->zstream) {
REFTABLE_CALLOC_ARRAY(br->zstream, 1);
if (!br->zstream) {
if (!block->zstream) {
REFTABLE_CALLOC_ARRAY(block->zstream, 1);
if (!block->zstream) {
err = REFTABLE_OUT_OF_MEMORY_ERROR;
goto done;
}

err = inflateInit(br->zstream);
err = inflateInit(block->zstream);
} else {
err = inflateReset(br->zstream);
err = inflateReset(block->zstream);
}
if (err != Z_OK) {
err = REFTABLE_ZLIB_ERROR;
goto done;
}

br->zstream->next_in = block->data + block_header_skip;
br->zstream->avail_in = src_len;
br->zstream->next_out = br->uncompressed_data + block_header_skip;
br->zstream->avail_out = dst_len;
block->zstream->next_in = block->block_data.data + block_header_skip;
block->zstream->avail_in = src_len;
block->zstream->next_out = block->uncompressed_data + block_header_skip;
block->zstream->avail_out = dst_len;

/*
* We know both input as well as output size, and we know that
@ -273,72 +299,71 @@ int block_reader_init(struct block_reader *br, struct reftable_block *block,
* here to instruct zlib to inflate the data in one go, which
* is more efficient than using `Z_NO_FLUSH`.
*/
err = inflate(br->zstream, Z_FINISH);
err = inflate(block->zstream, Z_FINISH);
if (err != Z_STREAM_END) {
err = REFTABLE_ZLIB_ERROR;
goto done;
}
err = 0;

if (br->zstream->total_out + block_header_skip != sz) {
if (block->zstream->total_out + block_header_skip != block_size) {
err = REFTABLE_FORMAT_ERROR;
goto done;
}

/* We're done with the input data. */
reftable_block_done(block);
block->data = br->uncompressed_data;
block->len = sz;
full_block_size = src_len + block_header_skip - br->zstream->avail_in;
block_source_release_data(&block->block_data);
block->block_data.data = block->uncompressed_data;
block->block_data.len = block_size;
full_block_size = src_len + block_header_skip - block->zstream->avail_in;
} else if (full_block_size == 0) {
full_block_size = sz;
} else if (sz < full_block_size && sz < block->len &&
block->data[sz] != 0) {
full_block_size = block_size;
} else if (block_size < full_block_size && block_size < block->block_data.len &&
block->block_data.data[block_size] != 0) {
/* If the block is smaller than the full block size, it is
padded (data followed by '\0') or the next block is
unaligned. */
full_block_size = sz;
full_block_size = block_size;
}

restart_count = reftable_get_be16(block->data + sz - 2);
restart_start = sz - 2 - 3 * restart_count;
restart_bytes = block->data + restart_start;
restart_count = reftable_get_be16(block->block_data.data + block_size - 2);
restart_off = block_size - 2 - 3 * restart_count;

/* transfer ownership. */
br->block = *block;
block->data = NULL;
block->len = 0;
block->block_type = block_type;
block->hash_size = hash_size;
block->restart_off = restart_off;
block->full_block_size = full_block_size;
block->header_off = header_size;
block->restart_count = restart_count;

br->hash_size = hash_size;
br->block_len = restart_start;
br->full_block_size = full_block_size;
br->header_off = header_off;
br->restart_count = restart_count;
br->restart_bytes = restart_bytes;
err = 0;

done:
if (err < 0)
reftable_block_release(block);
return err;
}

void block_reader_release(struct block_reader *br)
void reftable_block_release(struct reftable_block *block)
{
inflateEnd(br->zstream);
reftable_free(br->zstream);
reftable_free(br->uncompressed_data);
reftable_block_done(&br->block);
inflateEnd(block->zstream);
reftable_free(block->zstream);
reftable_free(block->uncompressed_data);
block_source_release_data(&block->block_data);
memset(block, 0, sizeof(*block));
}

uint8_t block_reader_type(const struct block_reader *r)
uint8_t reftable_block_type(const struct reftable_block *b)
{
return r->block.data[r->header_off];
return b->block_data.data[b->header_off];
}

int block_reader_first_key(const struct block_reader *br, struct reftable_buf *key)
int reftable_block_first_key(const struct reftable_block *block, struct reftable_buf *key)
{
int off = br->header_off + 4, n;
int off = block->header_off + 4, n;
struct string_view in = {
.buf = br->block.data + off,
.len = br->block_len - off,
.buf = block->block_data.data + off,
.len = block->restart_off - off,
};
uint8_t extra = 0;

@ -353,33 +378,36 @@ int block_reader_first_key(const struct block_reader *br, struct reftable_buf *k
return 0;
}

static uint32_t block_reader_restart_offset(const struct block_reader *br, size_t idx)
static uint32_t block_restart_offset(const struct reftable_block *b, size_t idx)
{
return reftable_get_be24(br->restart_bytes + 3 * idx);
return reftable_get_be24(b->block_data.data + b->restart_off + 3 * idx);
}

void block_iter_seek_start(struct block_iter *it, const struct block_reader *br)
void block_iter_init(struct block_iter *it, const struct reftable_block *block)
{
it->block = block;
block_iter_seek_start(it);
}

void block_iter_seek_start(struct block_iter *it)
{
it->block = br->block.data;
it->block_len = br->block_len;
it->hash_size = br->hash_size;
reftable_buf_reset(&it->last_key);
it->next_off = br->header_off + 4;
it->next_off = it->block->header_off + 4;
}

struct restart_needle_less_args {
int error;
struct reftable_buf needle;
const struct block_reader *reader;
const struct reftable_block *block;
};

static int restart_needle_less(size_t idx, void *_args)
{
struct restart_needle_less_args *args = _args;
uint32_t off = block_reader_restart_offset(args->reader, idx);
uint32_t off = block_restart_offset(args->block, idx);
struct string_view in = {
.buf = args->reader->block.data + off,
.len = args->reader->block_len - off,
.buf = args->block->block_data.data + off,
.len = args->block->restart_off - off,
};
uint64_t prefix_len, suffix_len;
uint8_t extra;
@ -412,14 +440,14 @@ static int restart_needle_less(size_t idx, void *_args)
int block_iter_next(struct block_iter *it, struct reftable_record *rec)
{
struct string_view in = {
.buf = (unsigned char *) it->block + it->next_off,
.len = it->block_len - it->next_off,
.buf = (unsigned char *) it->block->block_data.data + it->next_off,
.len = it->block->restart_off - it->next_off,
};
struct string_view start = in;
uint8_t extra = 0;
int n = 0;

if (it->next_off >= it->block_len)
if (it->next_off >= it->block->restart_off)
return 1;

n = reftable_decode_key(&it->last_key, &extra, in);
@ -429,7 +457,7 @@ int block_iter_next(struct block_iter *it, struct reftable_record *rec)
return REFTABLE_FORMAT_ERROR;

string_view_consume(&in, n);
n = reftable_record_decode(rec, it->last_key, extra, in, it->hash_size,
n = reftable_record_decode(rec, it->last_key, extra, in, it->block->hash_size,
&it->scratch);
if (n < 0)
return -1;
@ -444,8 +472,6 @@ void block_iter_reset(struct block_iter *it)
reftable_buf_reset(&it->last_key);
it->next_off = 0;
it->block = NULL;
it->block_len = 0;
it->hash_size = 0;
}

void block_iter_close(struct block_iter *it)
@ -454,12 +480,11 @@ void block_iter_close(struct block_iter *it)
reftable_buf_release(&it->scratch);
}

int block_iter_seek_key(struct block_iter *it, const struct block_reader *br,
struct reftable_buf *want)
int block_iter_seek_key(struct block_iter *it, struct reftable_buf *want)
{
struct restart_needle_less_args args = {
.needle = *want,
.reader = br,
.block = it->block,
};
struct reftable_record rec;
int err = 0;
@ -477,7 +502,7 @@ int block_iter_seek_key(struct block_iter *it, const struct block_reader *br,
* restart point. While that works alright, we would end up scanning
* too many record.
*/
i = binsearch(br->restart_count, &restart_needle_less, &args);
i = binsearch(it->block->restart_count, &restart_needle_less, &args);
if (args.error) {
err = REFTABLE_FORMAT_ERROR;
goto done;
@ -502,21 +527,18 @@ int block_iter_seek_key(struct block_iter *it, const struct block_reader *br,
* starting from the preceding restart point.
*/
if (i > 0)
it->next_off = block_reader_restart_offset(br, i - 1);
it->next_off = block_restart_offset(it->block, i - 1);
else
it->next_off = br->header_off + 4;
it->block = br->block.data;
it->block_len = br->block_len;
it->hash_size = br->hash_size;
it->next_off = it->block->header_off + 4;

err = reftable_record_init(&rec, block_reader_type(br));
err = reftable_record_init(&rec, reftable_block_type(it->block));
if (err < 0)
goto done;

/*
* We're looking for the last entry less than the wanted key so that
* the next call to `block_reader_next()` would yield the wanted
* record. We thus don't want to position our reader at the sought
* record. We thus don't want to position our iterator at the sought
* after record, but one before. To do so, we have to go one entry too
* far and then back up.
*/
@ -561,6 +583,61 @@ done:
return err;
}

static int block_iter_seek_void(void *it, struct reftable_record *want)
{
struct reftable_buf buf = REFTABLE_BUF_INIT;
struct block_iter *bi = it;
int err;

if (bi->block->block_type != want->type)
return REFTABLE_API_ERROR;

err = reftable_record_key(want, &buf);
if (err < 0)
goto out;

err = block_iter_seek_key(it, &buf);
if (err < 0)
goto out;

err = 0;

out:
reftable_buf_release(&buf);
return err;
}

static int block_iter_next_void(void *it, struct reftable_record *rec)
{
return block_iter_next(it, rec);
}

static void block_iter_close_void(void *it)
{
block_iter_close(it);
}

static struct reftable_iterator_vtable block_iter_vtable = {
.seek = &block_iter_seek_void,
.next = &block_iter_next_void,
.close = &block_iter_close_void,
};

int reftable_block_init_iterator(const struct reftable_block *b,
struct reftable_iterator *it)
{
struct block_iter *bi;

REFTABLE_CALLOC_ARRAY(bi, 1);
block_iter_init(bi, b);

assert(!it->ops);
it->iter_arg = bi;
it->ops = &block_iter_vtable;

return 0;
}

void block_writer_release(struct block_writer *bw)
{
deflateEnd(bw->zstream);
@ -571,14 +648,3 @@ void block_writer_release(struct block_writer *bw)
reftable_buf_release(&bw->last_key);
/* the block is not owned. */
}

void reftable_block_done(struct reftable_block *blockp)
{
struct reftable_block_source source = blockp->source;
if (blockp && source.ops)
source.ops->return_block(source.arg, blockp);
blockp->data = NULL;
blockp->len = 0;
blockp->source.ops = NULL;
blockp->source.arg = NULL;
}

85
reftable/block.h
View File

@ -1,16 +1,17 @@
/*
Copyright 2020 Google LLC

Use of this source code is governed by a BSD-style
license that can be found in the LICENSE file or at
https://developers.google.com/open-source/licenses/bsd
*/
* Copyright 2020 Google LLC
*
* Use of this source code is governed by a BSD-style
* license that can be found in the LICENSE file or at
* https://developers.google.com/open-source/licenses/bsd
*/

#ifndef BLOCK_H
#define BLOCK_H

#include "basics.h"
#include "record.h"
#include "reftable-block.h"
#include "reftable-blocksource.h"

/*
@ -18,7 +19,7 @@ https://developers.google.com/open-source/licenses/bsd
* allocation overhead.
*/
struct block_writer {
z_stream *zstream;
struct z_stream_s *zstream;
unsigned char *compressed;
size_t compressed_cap;

@ -62,53 +63,11 @@ int block_writer_finish(struct block_writer *w);
/* clears out internally allocated block_writer members. */
void block_writer_release(struct block_writer *bw);

struct z_stream;

/* Read a block. */
struct block_reader {
/* offset of the block header; nonzero for the first block in a
* reftable. */
uint32_t header_off;

/* the memory block */
struct reftable_block block;
uint32_t hash_size;

/* Uncompressed data for log entries. */
z_stream *zstream;
unsigned char *uncompressed_data;
size_t uncompressed_cap;

/* size of the data, excluding restart data. */
uint32_t block_len;
uint8_t *restart_bytes;
uint16_t restart_count;

/* size of the data in the file. For log blocks, this is the compressed
* size. */
uint32_t full_block_size;
};

/* initializes a block reader. */
int block_reader_init(struct block_reader *br, struct reftable_block *bl,
uint32_t header_off, uint32_t table_block_size,
uint32_t hash_size);

void block_reader_release(struct block_reader *br);

/* Returns the block type (eg. 'r' for refs) */
uint8_t block_reader_type(const struct block_reader *r);

/* Decodes the first key in the block */
int block_reader_first_key(const struct block_reader *br, struct reftable_buf *key);

/* Iterate over entries in a block */
/* Iterator for records contained in a single block. */
struct block_iter {
/* offset within the block of the next entry to read. */
uint32_t next_off;
const unsigned char *block;
size_t block_len;
uint32_t hash_size;
const struct reftable_block *block;

/* key for last entry we read. */
struct reftable_buf last_key;
@ -120,12 +79,23 @@ struct block_iter {
.scratch = REFTABLE_BUF_INIT, \
}

/* Position `it` at start of the block */
void block_iter_seek_start(struct block_iter *it, const struct block_reader *br);
/*
* Initialize the block iterator with the given block. The iterator will be
* positioned at the first record contained in the block. The block must remain
* valid until the end of the iterator's lifetime. It is valid to re-initialize
* iterators multiple times.
*/
void block_iter_init(struct block_iter *it, const struct reftable_block *block);

/* Position `it` to the `want` key in the block */
int block_iter_seek_key(struct block_iter *it, const struct block_reader *br,
struct reftable_buf *want);
/* Position the initialized iterator at the first record of its block. */
void block_iter_seek_start(struct block_iter *it);

/*
* Position the initialized iterator at the desired record key. It is not an
* error in case the record cannot be found. If so, a subsequent call to
* `block_iter_next()` will indicate that the iterator is exhausted.
*/
int block_iter_seek_key(struct block_iter *it, struct reftable_buf *want);

/* return < 0 for error, 0 for OK, > 0 for EOF. */
int block_iter_next(struct block_iter *it, struct reftable_record *rec);
@ -142,7 +112,4 @@ size_t header_size(int version);
/* size of file footer, depending on format version */
size_t footer_size(int version);

/* returns a block to its source. */
void reftable_block_done(struct reftable_block *ret);

#endif

67
reftable/blocksource.c
View File

@ -1,10 +1,10 @@
/*
Copyright 2020 Google LLC

Use of this source code is governed by a BSD-style
license that can be found in the LICENSE file or at
https://developers.google.com/open-source/licenses/bsd
*/
* Copyright 2020 Google LLC
*
* Use of this source code is governed by a BSD-style
* license that can be found in the LICENSE file or at
* https://developers.google.com/open-source/licenses/bsd
*/

#include "system.h"

@ -13,7 +13,42 @@ https://developers.google.com/open-source/licenses/bsd
#include "reftable-blocksource.h"
#include "reftable-error.h"

static void reftable_buf_return_block(void *b REFTABLE_UNUSED, struct reftable_block *dest)
void block_source_release_data(struct reftable_block_data *data)
{
struct reftable_block_source source = data->source;
if (data && source.ops)
source.ops->release_data(source.arg, data);
data->data = NULL;
data->len = 0;
data->source.ops = NULL;
data->source.arg = NULL;
}

void block_source_close(struct reftable_block_source *source)
{
if (!source->ops) {
return;
}

source->ops->close(source->arg);
source->ops = NULL;
}

ssize_t block_source_read_data(struct reftable_block_source *source,
struct reftable_block_data *dest, uint64_t off,
uint32_t size)
{
ssize_t result = source->ops->read_data(source->arg, dest, off, size);
dest->source = *source;
return result;
}

uint64_t block_source_size(struct reftable_block_source *source)
{
return source->ops->size(source->arg);
}

static void reftable_buf_release_data(void *b REFTABLE_UNUSED, struct reftable_block_data *dest)
{
if (dest->len)
memset(dest->data, 0xff, dest->len);
@ -24,8 +59,8 @@ static void reftable_buf_close(void *b REFTABLE_UNUSED)
{
}

static ssize_t reftable_buf_read_block(void *v, struct reftable_block *dest,
uint64_t off, uint32_t size)
static ssize_t reftable_buf_read_data(void *v, struct reftable_block_data *dest,
uint64_t off, uint32_t size)
{
struct reftable_buf *b = v;
assert(off + size <= b->len);
@ -44,8 +79,8 @@ static uint64_t reftable_buf_size(void *b)

static struct reftable_block_source_vtable reftable_buf_vtable = {
.size = &reftable_buf_size,
.read_block = &reftable_buf_read_block,
.return_block = &reftable_buf_return_block,
.read_data = &reftable_buf_read_data,
.release_data = &reftable_buf_release_data,
.close = &reftable_buf_close,
};

@ -67,7 +102,7 @@ static uint64_t file_size(void *b)
return ((struct file_block_source *)b)->size;
}

static void file_return_block(void *b REFTABLE_UNUSED, struct reftable_block *dest REFTABLE_UNUSED)
static void file_release_data(void *b REFTABLE_UNUSED, struct reftable_block_data *dest REFTABLE_UNUSED)
{
}

@ -78,8 +113,8 @@ static void file_close(void *v)
reftable_free(b);
}

static ssize_t file_read_block(void *v, struct reftable_block *dest, uint64_t off,
uint32_t size)
static ssize_t file_read_data(void *v, struct reftable_block_data *dest, uint64_t off,
uint32_t size)
{
struct file_block_source *b = v;
assert(off + size <= b->size);
@ -90,8 +125,8 @@ static ssize_t file_read_block(void *v, struct reftable_block *dest, uint64_t of

static struct reftable_block_source_vtable file_vtable = {
.size = &file_size,
.read_block = &file_read_block,
.return_block = &file_return_block,
.read_data = &file_read_data,
.release_data = &file_release_data,
.close = &file_close,
};


39
reftable/blocksource.h
View File

@ -1,10 +1,10 @@
/*
Copyright 2020 Google LLC

Use of this source code is governed by a BSD-style
license that can be found in the LICENSE file or at
https://developers.google.com/open-source/licenses/bsd
*/
* Copyright 2020 Google LLC
*
* Use of this source code is governed by a BSD-style
* license that can be found in the LICENSE file or at
* https://developers.google.com/open-source/licenses/bsd
*/

#ifndef BLOCKSOURCE_H
#define BLOCKSOURCE_H
@ -12,9 +12,34 @@ https://developers.google.com/open-source/licenses/bsd
#include "system.h"

struct reftable_block_source;
struct reftable_block_data;
struct reftable_buf;

/* Create an in-memory block source for reading reftables */
/*
* Close the block source and the underlying resource. This is a no-op in case
* the block source is zero-initialized.
*/
void block_source_close(struct reftable_block_source *source);

/*
* Read a block of length `size` from the source at the given `off`.
*/
ssize_t block_source_read_data(struct reftable_block_source *source,
struct reftable_block_data *dest, uint64_t off,
uint32_t size);

/*
* Return the total length of the underlying resource.
*/
uint64_t block_source_size(struct reftable_block_source *source);

/*
* Return a block to its original source, releasing any resources associated
* with it.
*/
void block_source_release_data(struct reftable_block_data *data);

/* Create an in-memory block source for reading reftables. */
void block_source_from_buf(struct reftable_block_source *bs,
struct reftable_buf *buf);


18
reftable/constants.h
View File

@ -1,19 +1,15 @@
/*
Copyright 2020 Google LLC

Use of this source code is governed by a BSD-style
license that can be found in the LICENSE file or at
https://developers.google.com/open-source/licenses/bsd
*/
* Copyright 2020 Google LLC
*
* Use of this source code is governed by a BSD-style
* license that can be found in the LICENSE file or at
* https://developers.google.com/open-source/licenses/bsd
*/

#ifndef CONSTANTS_H
#define CONSTANTS_H

#define BLOCK_TYPE_LOG 'g'
#define BLOCK_TYPE_INDEX 'i'
#define BLOCK_TYPE_REF 'r'
#define BLOCK_TYPE_OBJ 'o'
#define BLOCK_TYPE_ANY 0
#include "reftable-constants.h"

#define MAX_RESTARTS ((1 << 16) - 1)
#define DEFAULT_BLOCK_SIZE 4096

12
reftable/error.c
View File

@ -1,10 +1,10 @@
/*
Copyright 2020 Google LLC

Use of this source code is governed by a BSD-style
license that can be found in the LICENSE file or at
https://developers.google.com/open-source/licenses/bsd
*/
* Copyright 2020 Google LLC
*
* Use of this source code is governed by a BSD-style
* license that can be found in the LICENSE file or at
* https://developers.google.com/open-source/licenses/bsd
*/

#include "system.h"
#include "reftable-error.h"

36
reftable/iter.c
View File

@ -1,19 +1,20 @@
/*
Copyright 2020 Google LLC

Use of this source code is governed by a BSD-style
license that can be found in the LICENSE file or at
https://developers.google.com/open-source/licenses/bsd
*/
* Copyright 2020 Google LLC
*
* Use of this source code is governed by a BSD-style
* license that can be found in the LICENSE file or at
* https://developers.google.com/open-source/licenses/bsd
*/

#include "iter.h"

#include "system.h"

#include "block.h"
#include "blocksource.h"
#include "constants.h"
#include "reader.h"
#include "reftable-error.h"
#include "table.h"

int iterator_seek(struct reftable_iterator *it, struct reftable_record *want)
{
@ -113,7 +114,7 @@ static void indexed_table_ref_iter_close(void *p)
{
struct indexed_table_ref_iter *it = p;
block_iter_close(&it->cur);
reftable_block_done(&it->block_reader.block);
block_source_release_data(&it->block.block_data);
reftable_free(it->offsets);
reftable_buf_release(&it->oid);
}
@ -127,11 +128,10 @@ static int indexed_table_ref_iter_next_block(struct indexed_table_ref_iter *it)
return 1;
}

reftable_block_done(&it->block_reader.block);
block_source_release_data(&it->block.block_data);

off = it->offsets[it->offset_idx++];
err = reader_init_block_reader(it->r, &it->block_reader, off,
BLOCK_TYPE_REF);
err = table_init_block(it->table, &it->block, off, REFTABLE_BLOCK_TYPE_REF);
if (err < 0) {
return err;
}
@ -139,7 +139,7 @@ static int indexed_table_ref_iter_next_block(struct indexed_table_ref_iter *it)
/* indexed block does not exist. */
return REFTABLE_FORMAT_ERROR;
}
block_iter_seek_start(&it->cur, &it->block_reader);
block_iter_init(&it->cur, &it->block);
return 0;
}

@ -181,7 +181,7 @@ static int indexed_table_ref_iter_next(void *p, struct reftable_record *rec)
}

int indexed_table_ref_iter_new(struct indexed_table_ref_iter **dest,
struct reftable_reader *r, uint8_t *oid,
struct reftable_table *t, uint8_t *oid,
int oid_len, uint64_t *offsets, int offset_len)
{
struct indexed_table_ref_iter empty = INDEXED_TABLE_REF_ITER_INIT;
@ -195,7 +195,7 @@ int indexed_table_ref_iter_new(struct indexed_table_ref_iter **dest,
}

*itr = empty;
itr->r = r;
itr->table = t;

err = reftable_buf_add(&itr->oid, oid, oid_len);
if (err < 0)
@ -246,7 +246,7 @@ int reftable_iterator_seek_ref(struct reftable_iterator *it,
const char *name)
{
struct reftable_record want = {
.type = BLOCK_TYPE_REF,
.type = REFTABLE_BLOCK_TYPE_REF,
.u.ref = {
.refname = (char *)name,
},
@ -258,7 +258,7 @@ int reftable_iterator_next_ref(struct reftable_iterator *it,
struct reftable_ref_record *ref)
{
struct reftable_record rec = {
.type = BLOCK_TYPE_REF,
.type = REFTABLE_BLOCK_TYPE_REF,
.u = {
.ref = *ref
},
@ -272,7 +272,7 @@ int reftable_iterator_seek_log_at(struct reftable_iterator *it,
const char *name, uint64_t update_index)
{
struct reftable_record want = {
.type = BLOCK_TYPE_LOG,
.type = REFTABLE_BLOCK_TYPE_LOG,
.u.log = {
.refname = (char *)name,
.update_index = update_index,
@ -291,7 +291,7 @@ int reftable_iterator_next_log(struct reftable_iterator *it,
struct reftable_log_record *log)
{
struct reftable_record rec = {
.type = BLOCK_TYPE_LOG,
.type = REFTABLE_BLOCK_TYPE_LOG,
.u = {
.log = *log,
},

18
reftable/iter.h
View File

@ -1,10 +1,10 @@
/*
Copyright 2020 Google LLC

Use of this source code is governed by a BSD-style
license that can be found in the LICENSE file or at
https://developers.google.com/open-source/licenses/bsd
*/
* Copyright 2020 Google LLC
*
* Use of this source code is governed by a BSD-style
* license that can be found in the LICENSE file or at
* https://developers.google.com/open-source/licenses/bsd
*/

#ifndef ITER_H
#define ITER_H
@ -59,7 +59,7 @@ void iterator_from_filtering_ref_iterator(struct reftable_iterator *,
* but using the object index.
*/
struct indexed_table_ref_iter {
struct reftable_reader *r;
struct reftable_table *table;
struct reftable_buf oid;

/* mutable */
@ -68,7 +68,7 @@ struct indexed_table_ref_iter {
/* Points to the next offset to read. */
int offset_idx;
int offset_len;
struct block_reader block_reader;
struct reftable_block block;
struct block_iter cur;
int is_finished;
};
@ -83,7 +83,7 @@ void iterator_from_indexed_table_ref_iter(struct reftable_iterator *it,

/* Takes ownership of `offsets` */
int indexed_table_ref_iter_new(struct indexed_table_ref_iter **dest,
struct reftable_reader *r, uint8_t *oid,
struct reftable_table *t, uint8_t *oid,
int oid_len, uint64_t *offsets, int offset_len);

#endif

42
reftable/merged.c
View File

@ -1,21 +1,21 @@
/*
Copyright 2020 Google LLC

Use of this source code is governed by a BSD-style
license that can be found in the LICENSE file or at
https://developers.google.com/open-source/licenses/bsd
*/
* Copyright 2020 Google LLC
*
* Use of this source code is governed by a BSD-style
* license that can be found in the LICENSE file or at
* https://developers.google.com/open-source/licenses/bsd
*/

#include "merged.h"

#include "constants.h"
#include "iter.h"
#include "pq.h"
#include "reader.h"
#include "record.h"
#include "reftable-merged.h"
#include "reftable-error.h"
#include "system.h"
#include "table.h"

struct merged_subiter {
struct reftable_iterator iter;
@ -192,7 +192,7 @@ static void iterator_from_merged_iter(struct reftable_iterator *it,
}

int reftable_merged_table_new(struct reftable_merged_table **dest,
struct reftable_reader **readers, size_t n,
struct reftable_table **tables, size_t n,
enum reftable_hash hash_id)
{
struct reftable_merged_table *m = NULL;
@ -200,10 +200,10 @@ int reftable_merged_table_new(struct reftable_merged_table **dest,
uint64_t first_min = 0;

for (size_t i = 0; i < n; i++) {
uint64_t min = reftable_reader_min_update_index(readers[i]);
uint64_t max = reftable_reader_max_update_index(readers[i]);
uint64_t min = reftable_table_min_update_index(tables[i]);
uint64_t max = reftable_table_max_update_index(tables[i]);

if (reftable_reader_hash_id(readers[i]) != hash_id) {
if (reftable_table_hash_id(tables[i]) != hash_id) {
return REFTABLE_FORMAT_ERROR;
}
if (i == 0 || min < first_min) {
@ -218,8 +218,8 @@ int reftable_merged_table_new(struct reftable_merged_table **dest,
if (!m)
return REFTABLE_OUT_OF_MEMORY_ERROR;

m->readers = readers;
m->readers_len = n;
m->tables = tables;
m->tables_len = n;
m->min = first_min;
m->max = last_max;
m->hash_id = hash_id;
@ -254,20 +254,20 @@ int merged_table_init_iter(struct reftable_merged_table *mt,
struct merged_iter *mi = NULL;
int ret;

if (mt->readers_len) {
REFTABLE_CALLOC_ARRAY(subiters, mt->readers_len);
if (mt->tables_len) {
REFTABLE_CALLOC_ARRAY(subiters, mt->tables_len);
if (!subiters) {
ret = REFTABLE_OUT_OF_MEMORY_ERROR;
goto out;
}
}

for (size_t i = 0; i < mt->readers_len; i++) {
for (size_t i = 0; i < mt->tables_len; i++) {
ret = reftable_record_init(&subiters[i].rec, typ);
if (ret < 0)
goto out;

ret = reader_init_iter(mt->readers[i], &subiters[i].iter, typ);
ret = table_init_iter(mt->tables[i], &subiters[i].iter, typ);
if (ret < 0)
goto out;
}
@ -280,14 +280,14 @@ int merged_table_init_iter(struct reftable_merged_table *mt,
mi->advance_index = -1;
mi->suppress_deletions = mt->suppress_deletions;
mi->subiters = subiters;
mi->subiters_len = mt->readers_len;
mi->subiters_len = mt->tables_len;

iterator_from_merged_iter(it, mi);
ret = 0;

out:
if (ret < 0) {
for (size_t i = 0; subiters && i < mt->readers_len; i++) {
for (size_t i = 0; subiters && i < mt->tables_len; i++) {
reftable_iterator_destroy(&subiters[i].iter);
reftable_record_release(&subiters[i].rec);
}
@ -301,13 +301,13 @@ out:
int reftable_merged_table_init_ref_iterator(struct reftable_merged_table *mt,
struct reftable_iterator *it)
{
return merged_table_init_iter(mt, it, BLOCK_TYPE_REF);
return merged_table_init_iter(mt, it, REFTABLE_BLOCK_TYPE_REF);
}

int reftable_merged_table_init_log_iterator(struct reftable_merged_table *mt,
struct reftable_iterator *it)
{
return merged_table_init_iter(mt, it, BLOCK_TYPE_LOG);
return merged_table_init_iter(mt, it, REFTABLE_BLOCK_TYPE_LOG);
}

enum reftable_hash reftable_merged_table_hash_id(struct reftable_merged_table *mt)

16
reftable/merged.h
View File

@ -1,10 +1,10 @@
/*
Copyright 2020 Google LLC

Use of this source code is governed by a BSD-style
license that can be found in the LICENSE file or at
https://developers.google.com/open-source/licenses/bsd
*/
* Copyright 2020 Google LLC
*
* Use of this source code is governed by a BSD-style
* license that can be found in the LICENSE file or at
* https://developers.google.com/open-source/licenses/bsd
*/

#ifndef MERGED_H
#define MERGED_H
@ -13,8 +13,8 @@ https://developers.google.com/open-source/licenses/bsd
#include "reftable-basics.h"

struct reftable_merged_table {
struct reftable_reader **readers;
size_t readers_len;
struct reftable_table **tables;
size_t tables_len;
enum reftable_hash hash_id;

/* If unset, produce deletions. This is useful for compaction. For the

12
reftable/pq.c
View File

@ -1,10 +1,10 @@
/*
Copyright 2020 Google LLC

Use of this source code is governed by a BSD-style
license that can be found in the LICENSE file or at
https://developers.google.com/open-source/licenses/bsd
*/
* Copyright 2020 Google LLC
*
* Use of this source code is governed by a BSD-style
* license that can be found in the LICENSE file or at
* https://developers.google.com/open-source/licenses/bsd
*/

#include "pq.h"


12
reftable/pq.h
View File

@ -1,10 +1,10 @@
/*
Copyright 2020 Google LLC

Use of this source code is governed by a BSD-style
license that can be found in the LICENSE file or at
https://developers.google.com/open-source/licenses/bsd
*/
* Copyright 2020 Google LLC
*
* Use of this source code is governed by a BSD-style
* license that can be found in the LICENSE file or at
* https://developers.google.com/open-source/licenses/bsd
*/

#ifndef PQ_H
#define PQ_H

67
reftable/reader.h
View File

@ -1,67 +0,0 @@
/*
Copyright 2020 Google LLC

Use of this source code is governed by a BSD-style
license that can be found in the LICENSE file or at
https://developers.google.com/open-source/licenses/bsd
*/

#ifndef READER_H
#define READER_H

#include "block.h"
#include "record.h"
#include "reftable-iterator.h"
#include "reftable-reader.h"

uint64_t block_source_size(struct reftable_block_source *source);

ssize_t block_source_read_block(struct reftable_block_source *source,
struct reftable_block *dest, uint64_t off,
uint32_t size);
void block_source_close(struct reftable_block_source *source);

/* metadata for a block type */
struct reftable_reader_offsets {
int is_present;
uint64_t offset;
uint64_t index_offset;
};

/* The state for reading a reftable file. */
struct reftable_reader {
/* for convenience, associate a name with the instance. */
char *name;
struct reftable_block_source source;

/* Size of the file, excluding the footer. */
uint64_t size;

/* The hash function used for ref records. */
enum reftable_hash hash_id;

uint32_t block_size;
uint64_t min_update_index;
uint64_t max_update_index;
/* Length of the OID keys in the 'o' section */
int object_id_len;
int version;

struct reftable_reader_offsets ref_offsets;
struct reftable_reader_offsets obj_offsets;
struct reftable_reader_offsets log_offsets;

uint64_t refcount;
};

const char *reader_name(struct reftable_reader *r);

int reader_init_iter(struct reftable_reader *r,
struct reftable_iterator *it,
uint8_t typ);

/* initialize a block reader to read from `r` */
int reader_init_block_reader(struct reftable_reader *r, struct block_reader *br,
uint64_t next_off, uint8_t want_typ);

#endif

52
reftable/record.c
View File

@ -1,10 +1,10 @@
/*
Copyright 2020 Google LLC

Use of this source code is governed by a BSD-style
license that can be found in the LICENSE file or at
https://developers.google.com/open-source/licenses/bsd
*/
* Copyright 2020 Google LLC
*
* Use of this source code is governed by a BSD-style
* license that can be found in the LICENSE file or at
* https://developers.google.com/open-source/licenses/bsd
*/

/* record.c - methods for different types of records. */

@ -69,10 +69,10 @@ int put_var_int(struct string_view *dest, uint64_t value)
int reftable_is_block_type(uint8_t typ)
{
switch (typ) {
case BLOCK_TYPE_REF:
case BLOCK_TYPE_LOG:
case BLOCK_TYPE_OBJ:
case BLOCK_TYPE_INDEX:
case REFTABLE_BLOCK_TYPE_REF:
case REFTABLE_BLOCK_TYPE_LOG:
case REFTABLE_BLOCK_TYPE_OBJ:
case REFTABLE_BLOCK_TYPE_INDEX:
return 1;
}
return 0;
@ -459,7 +459,7 @@ static int reftable_ref_record_cmp_void(const void *_a, const void *_b)

static struct reftable_record_vtable reftable_ref_record_vtable = {
.key = &reftable_ref_record_key,
.type = BLOCK_TYPE_REF,
.type = REFTABLE_BLOCK_TYPE_REF,
.copy_from = &reftable_ref_record_copy_from,
.val_type = &reftable_ref_record_val_type,
.encode = &reftable_ref_record_encode,
@ -659,7 +659,7 @@ static int reftable_obj_record_cmp_void(const void *_a, const void *_b)

static struct reftable_record_vtable reftable_obj_record_vtable = {
.key = &reftable_obj_record_key,
.type = BLOCK_TYPE_OBJ,
.type = REFTABLE_BLOCK_TYPE_OBJ,
.copy_from = &reftable_obj_record_copy_from,
.val_type = &reftable_obj_record_val_type,
.encode = &reftable_obj_record_encode,
@ -1030,7 +1030,7 @@ static int reftable_log_record_is_deletion_void(const void *p)

static struct reftable_record_vtable reftable_log_record_vtable = {
.key = &reftable_log_record_key,
.type = BLOCK_TYPE_LOG,
.type = REFTABLE_BLOCK_TYPE_LOG,
.copy_from = &reftable_log_record_copy_from,
.val_type = &reftable_log_record_val_type,
.encode = &reftable_log_record_encode,
@ -1132,7 +1132,7 @@ static int reftable_index_record_cmp(const void *_a, const void *_b)

static struct reftable_record_vtable reftable_index_record_vtable = {
.key = &reftable_index_record_key,
.type = BLOCK_TYPE_INDEX,
.type = REFTABLE_BLOCK_TYPE_INDEX,
.copy_from = &reftable_index_record_copy_from,
.val_type = &reftable_index_record_val_type,
.encode = &reftable_index_record_encode,
@ -1275,13 +1275,13 @@ int reftable_log_record_is_deletion(const struct reftable_log_record *log)
static void *reftable_record_data(struct reftable_record *rec)
{
switch (rec->type) {
case BLOCK_TYPE_REF:
case REFTABLE_BLOCK_TYPE_REF:
return &rec->u.ref;
case BLOCK_TYPE_LOG:
case REFTABLE_BLOCK_TYPE_LOG:
return &rec->u.log;
case BLOCK_TYPE_INDEX:
case REFTABLE_BLOCK_TYPE_INDEX:
return &rec->u.idx;
case BLOCK_TYPE_OBJ:
case REFTABLE_BLOCK_TYPE_OBJ:
return &rec->u.obj;
}
abort();
@ -1291,13 +1291,13 @@ static struct reftable_record_vtable *
reftable_record_vtable(struct reftable_record *rec)
{
switch (rec->type) {
case BLOCK_TYPE_REF:
case REFTABLE_BLOCK_TYPE_REF:
return &reftable_ref_record_vtable;
case BLOCK_TYPE_LOG:
case REFTABLE_BLOCK_TYPE_LOG:
return &reftable_log_record_vtable;
case BLOCK_TYPE_INDEX:
case REFTABLE_BLOCK_TYPE_INDEX:
return &reftable_index_record_vtable;
case BLOCK_TYPE_OBJ:
case REFTABLE_BLOCK_TYPE_OBJ:
return &reftable_obj_record_vtable;
}
abort();
@ -1309,11 +1309,11 @@ int reftable_record_init(struct reftable_record *rec, uint8_t typ)
rec->type = typ;

switch (typ) {
case BLOCK_TYPE_REF:
case BLOCK_TYPE_LOG:
case BLOCK_TYPE_OBJ:
case REFTABLE_BLOCK_TYPE_REF:
case REFTABLE_BLOCK_TYPE_LOG:
case REFTABLE_BLOCK_TYPE_OBJ:
return 0;
case BLOCK_TYPE_INDEX:
case REFTABLE_BLOCK_TYPE_INDEX:
reftable_buf_init(&rec->u.idx.last_key);
return 0;
default:

12
reftable/record.h
View File

@ -1,10 +1,10 @@
/*
Copyright 2020 Google LLC

Use of this source code is governed by a BSD-style
license that can be found in the LICENSE file or at
https://developers.google.com/open-source/licenses/bsd
*/
* Copyright 2020 Google LLC
*
* Use of this source code is governed by a BSD-style
* license that can be found in the LICENSE file or at
* https://developers.google.com/open-source/licenses/bsd
*/

#ifndef RECORD_H
#define RECORD_H

10
reftable/reftable-basics.h
View File

@ -4,13 +4,21 @@
* Use of this source code is governed by a BSD-style
* license that can be found in the LICENSE file or at
* https://developers.google.com/open-source/licenses/bsd
*/
*/

#ifndef REFTABLE_BASICS_H
#define REFTABLE_BASICS_H

#include <stddef.h>

/* A buffer that contains arbitrary byte slices. */
struct reftable_buf {
size_t alloc;
size_t len;
char *buf;
};
#define REFTABLE_BUF_INIT { 0 }

/*
* Hash functions understood by the reftable library. Note that the values are
* arbitrary and somewhat random such that we can easily detect cases where the

74
reftable/reftable-block.h Normal file
View File

@ -0,0 +1,74 @@
/*
* Copyright 2020 Google LLC
*
* Use of this source code is governed by a BSD-style
* license that can be found in the LICENSE file or at
* https://developers.google.com/open-source/licenses/bsd
*/

#ifndef REFTABLE_BLOCK_H
#define REFTABLE_BLOCK_H

#include <stdint.h>

#include "reftable-basics.h"
#include "reftable-blocksource.h"
#include "reftable-iterator.h"

struct z_stream_s;

/*
* A block part of a reftable. Contains records as well as some metadata
* describing them.
*/
struct reftable_block {
/*
* Offset of the block header; nonzero for the first block in a
* reftable.
*/
uint32_t header_off;

/* The memory block. */
struct reftable_block_data block_data;
uint32_t hash_size;

/* Uncompressed data for log entries. */
struct z_stream_s *zstream;
unsigned char *uncompressed_data;
size_t uncompressed_cap;

/*
* Restart point data. Restart points are located after the block's
* record data.
*/
uint16_t restart_count;
uint32_t restart_off;

/*
* Size of the data in the file. For log blocks, this is the compressed
* size.
*/
uint32_t full_block_size;
uint8_t block_type;
};

/* Initialize a reftable block from the given block source. */
int reftable_block_init(struct reftable_block *b,
struct reftable_block_source *source,
uint32_t offset, uint32_t header_size,
uint32_t table_block_size, uint32_t hash_size);

/* Release resources allocated by the block. */
void reftable_block_release(struct reftable_block *b);

/* Initialize a generic record iterator from the given block. */
int reftable_block_init_iterator(const struct reftable_block *b,
struct reftable_iterator *it);

/* Returns the block type (eg. 'r' for refs). */
uint8_t reftable_block_type(const struct reftable_block *b);

/* Decodes the first key in the block. */
int reftable_block_first_key(const struct reftable_block *b, struct reftable_buf *key);

#endif /* REFTABLE_BLOCK_H */

29
reftable/reftable-blocksource.h
View File

@ -1,17 +1,18 @@
/*
Copyright 2020 Google LLC

Use of this source code is governed by a BSD-style
license that can be found in the LICENSE file or at
https://developers.google.com/open-source/licenses/bsd
*/
* Copyright 2020 Google LLC
*
* Use of this source code is governed by a BSD-style
* license that can be found in the LICENSE file or at
* https://developers.google.com/open-source/licenses/bsd
*/

#ifndef REFTABLE_BLOCKSOURCE_H
#define REFTABLE_BLOCKSOURCE_H

#include <stdint.h>

/* block_source is a generic wrapper for a seekable readable file.
/*
* Generic wrapper for a seekable readable file.
*/
struct reftable_block_source {
struct reftable_block_source_vtable *ops;
@ -20,7 +21,7 @@ struct reftable_block_source {

/* a contiguous segment of bytes. It keeps track of its generating block_source
* so it can return itself into the pool. */
struct reftable_block {
struct reftable_block_data {
uint8_t *data;
size_t len;
struct reftable_block_source source;
@ -28,20 +29,20 @@ struct reftable_block {

/* block_source_vtable are the operations that make up block_source */
struct reftable_block_source_vtable {
/* returns the size of a block source */
/* Returns the size of a block source. */
uint64_t (*size)(void *source);

/*
* Reads a segment from the block source. It is an error to read beyond
* the end of the block.
*/
ssize_t (*read_block)(void *source, struct reftable_block *dest,
uint64_t off, uint32_t size);
ssize_t (*read_data)(void *source, struct reftable_block_data *dest,
uint64_t off, uint32_t size);

/* mark the block as read; may return the data back to malloc */
void (*return_block)(void *source, struct reftable_block *blockp);
/* Mark the block as read; may release the data. */
void (*release_data)(void *source, struct reftable_block_data *data);

/* release all resources associated with the block source */
/* Release all resources associated with the block source. */
void (*close)(void *source);
};


18
reftable/reftable-constants.h Normal file
View File

@ -0,0 +1,18 @@
/*
* Copyright 2020 Google LLC
*
* Use of this source code is governed by a BSD-style
* license that can be found in the LICENSE file or at
* https://developers.google.com/open-source/licenses/bsd
*/

#ifndef REFTABLE_CONSTANTS_H
#define REFTABLE_CONSTANTS_H

#define REFTABLE_BLOCK_TYPE_LOG 'g'
#define REFTABLE_BLOCK_TYPE_INDEX 'i'
#define REFTABLE_BLOCK_TYPE_REF 'r'
#define REFTABLE_BLOCK_TYPE_OBJ 'o'
#define REFTABLE_BLOCK_TYPE_ANY 0

#endif /* REFTABLE_CONSTANTS_H */

12
reftable/reftable-error.h
View File

@ -1,10 +1,10 @@
/*
Copyright 2020 Google LLC

Use of this source code is governed by a BSD-style
license that can be found in the LICENSE file or at
https://developers.google.com/open-source/licenses/bsd
*/
* Copyright 2020 Google LLC
*
* Use of this source code is governed by a BSD-style
* license that can be found in the LICENSE file or at
* https://developers.google.com/open-source/licenses/bsd
*/

#ifndef REFTABLE_ERROR_H
#define REFTABLE_ERROR_H

12
reftable/reftable-iterator.h
View File

@ -1,10 +1,10 @@
/*
Copyright 2020 Google LLC

Use of this source code is governed by a BSD-style
license that can be found in the LICENSE file or at
https://developers.google.com/open-source/licenses/bsd
*/
* Copyright 2020 Google LLC
*
* Use of this source code is governed by a BSD-style
* license that can be found in the LICENSE file or at
* https://developers.google.com/open-source/licenses/bsd
*/

#ifndef REFTABLE_ITERATOR_H
#define REFTABLE_ITERATOR_H

18
reftable/reftable-merged.h
View File

@ -1,10 +1,10 @@
/*
Copyright 2020 Google LLC

Use of this source code is governed by a BSD-style
license that can be found in the LICENSE file or at
https://developers.google.com/open-source/licenses/bsd
*/
* Copyright 2020 Google LLC
*
* Use of this source code is governed by a BSD-style
* license that can be found in the LICENSE file or at
* https://developers.google.com/open-source/licenses/bsd
*/

#ifndef REFTABLE_MERGED_H
#define REFTABLE_MERGED_H
@ -26,14 +26,14 @@ https://developers.google.com/open-source/licenses/bsd
/* A merged table is implements seeking/iterating over a stack of tables. */
struct reftable_merged_table;

struct reftable_reader;
struct reftable_table;

/*
* reftable_merged_table_new creates a new merged table. The readers must be
* reftable_merged_table_new creates a new merged table. The tables must be
* kept alive as long as the merged table is still in use.
*/
int reftable_merged_table_new(struct reftable_merged_table **dest,
struct reftable_reader **readers, size_t n,
struct reftable_table **tables, size_t n,
enum reftable_hash hash_id);

/* Initialize a merged table iterator for reading refs. */

72
reftable/reftable-reader.h
View File

@ -1,72 +0,0 @@
/*
Copyright 2020 Google LLC

Use of this source code is governed by a BSD-style
license that can be found in the LICENSE file or at
https://developers.google.com/open-source/licenses/bsd
*/

#ifndef REFTABLE_READER_H
#define REFTABLE_READER_H

#include "reftable-iterator.h"
#include "reftable-blocksource.h"

/*
* Reading single tables
*
* The follow routines are for reading single files. For an
* application-level interface, skip ahead to struct
* reftable_merged_table and struct reftable_stack.
*/

/* The reader struct is a handle to an open reftable file. */
struct reftable_reader;

/* reftable_reader_new opens a reftable for reading. If successful,
* returns 0 code and sets pp. The name is used for creating a
* stack. Typically, it is the basename of the file. The block source
* `src` is owned by the reader, and is closed on calling
* reftable_reader_destroy(). On error, the block source `src` is
* closed as well.
*/
int reftable_reader_new(struct reftable_reader **pp,
struct reftable_block_source *src, const char *name);

/*
* Manage the reference count of the reftable reader. A newly initialized
* reader starts with a refcount of 1 and will be deleted once the refcount has
* reached 0.
*
* This is required because readers may have longer lifetimes than the stack
* they belong to. The stack may for example be reloaded while the old tables
* are still being accessed by an iterator.
*/
void reftable_reader_incref(struct reftable_reader *reader);
void reftable_reader_decref(struct reftable_reader *reader);

/* Initialize a reftable iterator for reading refs. */
int reftable_reader_init_ref_iterator(struct reftable_reader *r,
struct reftable_iterator *it);

/* Initialize a reftable iterator for reading logs. */
int reftable_reader_init_log_iterator(struct reftable_reader *r,
struct reftable_iterator *it);

/* returns the hash ID used in this table. */
enum reftable_hash reftable_reader_hash_id(struct reftable_reader *r);

/* return an iterator for the refs pointing to `oid`. */
int reftable_reader_refs_for(struct reftable_reader *r,
struct reftable_iterator *it, uint8_t *oid);

/* return the max_update_index for a table */
uint64_t reftable_reader_max_update_index(struct reftable_reader *r);

/* return the min_update_index for a table */
uint64_t reftable_reader_min_update_index(struct reftable_reader *r);

/* print blocks onto stdout for debugging. */
int reftable_reader_print_blocks(const char *tablename);

#endif

12
reftable/reftable-record.h
View File

@ -1,10 +1,10 @@
/*
Copyright 2020 Google LLC

Use of this source code is governed by a BSD-style
license that can be found in the LICENSE file or at
https://developers.google.com/open-source/licenses/bsd
*/
* Copyright 2020 Google LLC
*
* Use of this source code is governed by a BSD-style
* license that can be found in the LICENSE file or at
* https://developers.google.com/open-source/licenses/bsd
*/

#ifndef REFTABLE_RECORD_H
#define REFTABLE_RECORD_H

12
reftable/reftable-stack.h
View File

@ -1,10 +1,10 @@
/*
Copyright 2020 Google LLC

Use of this source code is governed by a BSD-style
license that can be found in the LICENSE file or at
https://developers.google.com/open-source/licenses/bsd
*/
* Copyright 2020 Google LLC
*
* Use of this source code is governed by a BSD-style
* license that can be found in the LICENSE file or at
* https://developers.google.com/open-source/licenses/bsd
*/

#ifndef REFTABLE_STACK_H
#define REFTABLE_STACK_H

115
reftable/reftable-table.h Normal file
View File

@ -0,0 +1,115 @@
/*
* Copyright 2020 Google LLC
*
* Use of this source code is governed by a BSD-style
* license that can be found in the LICENSE file or at
* https://developers.google.com/open-source/licenses/bsd
*/

#ifndef REFTABLE_TABLE_H
#define REFTABLE_TABLE_H

#include "reftable-iterator.h"
#include "reftable-block.h"
#include "reftable-blocksource.h"

/*
* Reading single tables
*
* The follow routines are for reading single files. For an
* application-level interface, skip ahead to struct
* reftable_merged_table and struct reftable_stack.
*/

/* Metadata for a block type. */
struct reftable_table_offsets {
int is_present;
uint64_t offset;
uint64_t index_offset;
};

/* The table struct is a handle to an open reftable file. */
struct reftable_table {
/* for convenience, associate a name with the instance. */
char *name;
struct reftable_block_source source;

/* Size of the file, excluding the footer. */
uint64_t size;

/* The hash function used for ref records. */
enum reftable_hash hash_id;

uint32_t block_size;
uint64_t min_update_index;
uint64_t max_update_index;
/* Length of the OID keys in the 'o' section */
int object_id_len;
int version;

struct reftable_table_offsets ref_offsets;
struct reftable_table_offsets obj_offsets;
struct reftable_table_offsets log_offsets;

uint64_t refcount;
};

/* reftable_table_new opens a reftable for reading. If successful,
* returns 0 code and sets pp. The name is used for creating a
* stack. Typically, it is the basename of the file. The block source
* `src` is owned by the table, and is closed on calling
* reftable_table_destroy(). On error, the block source `src` is
* closed as well.
*/
int reftable_table_new(struct reftable_table **out,
struct reftable_block_source *src, const char *name);

/*
* Manage the reference count of the reftable table. A newly initialized
* table starts with a refcount of 1 and will be deleted once the refcount has
* reached 0.
*
* This is required because tables may have longer lifetimes than the stack
* they belong to. The stack may for example be reloaded while the old tables
* are still being accessed by an iterator.
*/
void reftable_table_incref(struct reftable_table *table);
void reftable_table_decref(struct reftable_table *table);

/* Initialize a reftable iterator for reading refs. */
int reftable_table_init_ref_iterator(struct reftable_table *t,
struct reftable_iterator *it);

/* Initialize a reftable iterator for reading logs. */
int reftable_table_init_log_iterator(struct reftable_table *t,
struct reftable_iterator *it);

/* returns the hash ID used in this table. */
enum reftable_hash reftable_table_hash_id(struct reftable_table *t);

/* return an iterator for the refs pointing to `oid`. */
int reftable_table_refs_for(struct reftable_table *t,
struct reftable_iterator *it, uint8_t *oid);

/* return the max_update_index for a table */
uint64_t reftable_table_max_update_index(struct reftable_table *t);

/* return the min_update_index for a table */
uint64_t reftable_table_min_update_index(struct reftable_table *t);

/*
* An iterator that iterates through the blocks contained in a given table.
*/
struct reftable_table_iterator {
void *iter_arg;
};

int reftable_table_iterator_init(struct reftable_table_iterator *it,
struct reftable_table *t);

void reftable_table_iterator_release(struct reftable_table_iterator *it);

int reftable_table_iterator_next(struct reftable_table_iterator *it,
const struct reftable_block **out);

#endif

12
reftable/reftable-writer.h
View File

@ -1,10 +1,10 @@
/*
Copyright 2020 Google LLC

Use of this source code is governed by a BSD-style
license that can be found in the LICENSE file or at
https://developers.google.com/open-source/licenses/bsd
*/
* Copyright 2020 Google LLC
*
* Use of this source code is governed by a BSD-style
* license that can be found in the LICENSE file or at
* https://developers.google.com/open-source/licenses/bsd
*/

#ifndef REFTABLE_WRITER_H
#define REFTABLE_WRITER_H

188
reftable/stack.c
View File

@ -1,20 +1,20 @@
/*
Copyright 2020 Google LLC

Use of this source code is governed by a BSD-style
license that can be found in the LICENSE file or at
https://developers.google.com/open-source/licenses/bsd
*/
* Copyright 2020 Google LLC
*
* Use of this source code is governed by a BSD-style
* license that can be found in the LICENSE file or at
* https://developers.google.com/open-source/licenses/bsd
*/

#include "stack.h"

#include "system.h"
#include "constants.h"
#include "merged.h"
#include "reader.h"
#include "reftable-error.h"
#include "reftable-record.h"
#include "reftable-merged.h"
#include "table.h"
#include "writer.h"

static int stack_try_add(struct reftable_stack *st,
@ -203,14 +203,14 @@ int reftable_stack_init_ref_iterator(struct reftable_stack *st,
struct reftable_iterator *it)
{
return merged_table_init_iter(reftable_stack_merged_table(st),
it, BLOCK_TYPE_REF);
it, REFTABLE_BLOCK_TYPE_REF);
}

int reftable_stack_init_log_iterator(struct reftable_stack *st,
struct reftable_iterator *it)
{
return merged_table_init_iter(reftable_stack_merged_table(st),
it, BLOCK_TYPE_LOG);
it, REFTABLE_BLOCK_TYPE_LOG);
}

struct reftable_merged_table *
@ -248,11 +248,11 @@ void reftable_stack_destroy(struct reftable_stack *st)
REFTABLE_FREE_AND_NULL(names);
}

if (st->readers) {
if (st->tables) {
struct reftable_buf filename = REFTABLE_BUF_INIT;

for (size_t i = 0; i < st->readers_len; i++) {
const char *name = reader_name(st->readers[i]);
for (size_t i = 0; i < st->tables_len; i++) {
const char *name = reftable_table_name(st->tables[i]);
int try_unlinking = 1;

reftable_buf_reset(&filename);
@ -260,7 +260,7 @@ void reftable_stack_destroy(struct reftable_stack *st)
if (stack_filename(&filename, st, name) < 0)
try_unlinking = 0;
}
reftable_reader_decref(st->readers[i]);
reftable_table_decref(st->tables[i]);

if (try_unlinking && filename.len) {
/* On Windows, can only unlink after closing. */
@ -269,8 +269,8 @@ void reftable_stack_destroy(struct reftable_stack *st)
}

reftable_buf_release(&filename);
st->readers_len = 0;
REFTABLE_FREE_AND_NULL(st->readers);
st->tables_len = 0;
REFTABLE_FREE_AND_NULL(st->tables);
}

if (st->list_fd >= 0) {
@ -284,14 +284,14 @@ void reftable_stack_destroy(struct reftable_stack *st)
free_names(names);
}

static struct reftable_reader **stack_copy_readers(struct reftable_stack *st,
size_t cur_len)
static struct reftable_table **stack_copy_tables(struct reftable_stack *st,
size_t cur_len)
{
struct reftable_reader **cur = reftable_calloc(cur_len, sizeof(*cur));
struct reftable_table **cur = reftable_calloc(cur_len, sizeof(*cur));
if (!cur)
return NULL;
for (size_t i = 0; i < cur_len; i++)
cur[i] = st->readers[i];
cur[i] = st->tables[i];
return cur;
}

@ -299,19 +299,19 @@ static int reftable_stack_reload_once(struct reftable_stack *st,
const char **names,
int reuse_open)
{
size_t cur_len = !st->merged ? 0 : st->merged->readers_len;
struct reftable_reader **cur = NULL;
struct reftable_reader **reused = NULL;
struct reftable_reader **new_readers = NULL;
size_t cur_len = !st->merged ? 0 : st->merged->tables_len;
struct reftable_table **cur = NULL;
struct reftable_table **reused = NULL;
struct reftable_table **new_tables = NULL;
size_t reused_len = 0, reused_alloc = 0, names_len;
size_t new_readers_len = 0;
size_t new_tables_len = 0;
struct reftable_merged_table *new_merged = NULL;
struct reftable_buf table_path = REFTABLE_BUF_INIT;
int err = 0;
size_t i;

if (cur_len) {
cur = stack_copy_readers(st, cur_len);
cur = stack_copy_tables(st, cur_len);
if (!cur) {
err = REFTABLE_OUT_OF_MEMORY_ERROR;
goto done;
@ -321,28 +321,28 @@ static int reftable_stack_reload_once(struct reftable_stack *st,
names_len = names_length(names);

if (names_len) {
new_readers = reftable_calloc(names_len, sizeof(*new_readers));
if (!new_readers) {
new_tables = reftable_calloc(names_len, sizeof(*new_tables));
if (!new_tables) {
err = REFTABLE_OUT_OF_MEMORY_ERROR;
goto done;
}
}

while (*names) {
struct reftable_reader *rd = NULL;
struct reftable_table *table = NULL;
const char *name = *names++;

/* this is linear; we assume compaction keeps the number of
tables under control so this is not quadratic. */
for (i = 0; reuse_open && i < cur_len; i++) {
if (cur[i] && 0 == strcmp(cur[i]->name, name)) {
rd = cur[i];
table = cur[i];
cur[i] = NULL;

/*
* When reloading the stack fails, we end up
* releasing all new readers. This also
* includes the reused readers, even though
* releasing all new tables. This also
* includes the reused tables, even though
* they are still in used by the old stack. We
* thus need to keep them alive here, which we
* do by bumping their refcount.
@ -354,13 +354,13 @@ static int reftable_stack_reload_once(struct reftable_stack *st,
err = REFTABLE_OUT_OF_MEMORY_ERROR;
goto done;
}
reused[reused_len++] = rd;
reftable_reader_incref(rd);
reused[reused_len++] = table;
reftable_table_incref(table);
break;
}
}

if (!rd) {
if (!table) {
struct reftable_block_source src = { NULL };

err = stack_filename(&table_path, st, name);
@ -372,36 +372,36 @@ static int reftable_stack_reload_once(struct reftable_stack *st,
if (err < 0)
goto done;

err = reftable_reader_new(&rd, &src, name);
err = reftable_table_new(&table, &src, name);
if (err < 0)
goto done;
}

new_readers[new_readers_len] = rd;
new_readers_len++;
new_tables[new_tables_len] = table;
new_tables_len++;
}

/* success! */
err = reftable_merged_table_new(&new_merged, new_readers,
new_readers_len, st->opts.hash_id);
err = reftable_merged_table_new(&new_merged, new_tables,
new_tables_len, st->opts.hash_id);
if (err < 0)
goto done;

/*
* Close the old, non-reused readers and proactively try to unlink
* Close the old, non-reused tables and proactively try to unlink
* them. This is done for systems like Windows, where the underlying
* file of such an open reader wouldn't have been possible to be
* file of such an open table wouldn't have been possible to be
* unlinked by the compacting process.
*/
for (i = 0; i < cur_len; i++) {
if (cur[i]) {
const char *name = reader_name(cur[i]);
const char *name = reftable_table_name(cur[i]);

err = stack_filename(&table_path, st, name);
if (err < 0)
goto done;

reftable_reader_decref(cur[i]);
reftable_table_decref(cur[i]);
unlink(table_path.buf);
}
}
@ -412,25 +412,25 @@ static int reftable_stack_reload_once(struct reftable_stack *st,
new_merged->suppress_deletions = 1;
st->merged = new_merged;

if (st->readers)
reftable_free(st->readers);
st->readers = new_readers;
st->readers_len = new_readers_len;
new_readers = NULL;
new_readers_len = 0;
if (st->tables)
reftable_free(st->tables);
st->tables = new_tables;
st->tables_len = new_tables_len;
new_tables = NULL;
new_tables_len = 0;

/*
* Decrement the refcount of reused readers again. This only needs to
* Decrement the refcount of reused tables again. This only needs to
* happen on the successful case, because on the unsuccessful one we
* decrement their refcount via `new_readers`.
* decrement their refcount via `new_tables`.
*/
for (i = 0; i < reused_len; i++)
reftable_reader_decref(reused[i]);
reftable_table_decref(reused[i]);

done:
for (i = 0; i < new_readers_len; i++)
reftable_reader_decref(new_readers[i]);
reftable_free(new_readers);
for (i = 0; i < new_tables_len; i++)
reftable_table_decref(new_tables[i]);
reftable_free(new_tables);
reftable_free(reused);
reftable_free(cur);
reftable_buf_release(&table_path);
@ -615,10 +615,10 @@ static int stack_uptodate(struct reftable_stack *st)
/*
* It's fine for "tables.list" to not exist. In that
* case, we have to refresh when the loaded stack has
* any readers.
* any tables.
*/
if (errno == ENOENT)
return !!st->readers_len;
return !!st->tables_len;
return REFTABLE_IO_ERROR;
}

@ -637,19 +637,19 @@ static int stack_uptodate(struct reftable_stack *st)
if (err < 0)
return err;

for (size_t i = 0; i < st->readers_len; i++) {
for (size_t i = 0; i < st->tables_len; i++) {
if (!names[i]) {
err = 1;
goto done;
}

if (strcmp(st->readers[i]->name, names[i])) {
if (strcmp(st->tables[i]->name, names[i])) {
err = 1;
goto done;
}
}

if (names[st->merged->readers_len]) {
if (names[st->merged->tables_len]) {
err = 1;
goto done;
}
@ -792,8 +792,8 @@ int reftable_addition_commit(struct reftable_addition *add)
if (add->new_tables_len == 0)
goto done;

for (i = 0; i < add->stack->merged->readers_len; i++) {
if ((err = reftable_buf_addstr(&table_list, add->stack->readers[i]->name)) < 0 ||
for (i = 0; i < add->stack->merged->tables_len; i++) {
if ((err = reftable_buf_addstr(&table_list, add->stack->tables[i]->name)) < 0 ||
(err = reftable_buf_addstr(&table_list, "\n")) < 0)
goto done;
}
@ -1000,9 +1000,9 @@ done:

uint64_t reftable_stack_next_update_index(struct reftable_stack *st)
{
int sz = st->merged->readers_len;
int sz = st->merged->tables_len;
if (sz > 0)
return reftable_reader_max_update_index(st->readers[sz - 1]) +
return reftable_table_max_update_index(st->tables[sz - 1]) +
1;
return 1;
}
@ -1021,8 +1021,8 @@ static int stack_compact_locked(struct reftable_stack *st,
struct reftable_tmpfile tab_file = REFTABLE_TMPFILE_INIT;
int err = 0;

err = format_name(&next_name, reftable_reader_min_update_index(st->readers[first]),
reftable_reader_max_update_index(st->readers[last]));
err = format_name(&next_name, reftable_table_min_update_index(st->tables[first]),
reftable_table_max_update_index(st->tables[last]));
if (err < 0)
goto done;

@ -1087,18 +1087,18 @@ static int stack_write_compact(struct reftable_stack *st,
int err = 0;

for (size_t i = first; i <= last; i++)
st->stats.bytes += st->readers[i]->size;
err = reftable_writer_set_limits(wr, st->readers[first]->min_update_index,
st->readers[last]->max_update_index);
st->stats.bytes += st->tables[i]->size;
err = reftable_writer_set_limits(wr, st->tables[first]->min_update_index,
st->tables[last]->max_update_index);
if (err < 0)
goto done;

err = reftable_merged_table_new(&mt, st->readers + first, subtabs_len,
err = reftable_merged_table_new(&mt, st->tables + first, subtabs_len,
st->opts.hash_id);
if (err < 0)
goto done;

err = merged_table_init_iter(mt, &it, BLOCK_TYPE_REF);
err = merged_table_init_iter(mt, &it, REFTABLE_BLOCK_TYPE_REF);
if (err < 0)
goto done;

@ -1126,7 +1126,7 @@ static int stack_write_compact(struct reftable_stack *st,
}
reftable_iterator_destroy(&it);

err = merged_table_init_iter(mt, &it, BLOCK_TYPE_LOG);
err = merged_table_init_iter(mt, &it, REFTABLE_BLOCK_TYPE_LOG);
if (err < 0)
goto done;

@ -1250,7 +1250,7 @@ static int stack_compact_range(struct reftable_stack *st,
table_locks[i] = REFTABLE_FLOCK_INIT;

for (i = last + 1; i > first; i--) {
err = stack_filename(&table_name, st, reader_name(st->readers[i - 1]));
err = stack_filename(&table_name, st, reftable_table_name(st->tables[i - 1]));
if (err < 0)
goto done;

@ -1376,7 +1376,7 @@ static int stack_compact_range(struct reftable_stack *st,
* compacted in the updated "tables.list" file.
*/
for (size_t i = 0; names[i]; i++) {
if (strcmp(names[i], st->readers[first]->name))
if (strcmp(names[i], st->tables[first]->name))
continue;

/*
@ -1386,8 +1386,8 @@ static int stack_compact_range(struct reftable_stack *st,
* have compacted them.
*/
for (size_t j = 1; j < last - first + 1; j++) {
const char *old = first + j < st->merged->readers_len ?
st->readers[first + j]->name : NULL;
const char *old = first + j < st->merged->tables_len ?
st->tables[first + j]->name : NULL;
const char *new = names[i + j];

/*
@ -1427,16 +1427,16 @@ static int stack_compact_range(struct reftable_stack *st,
* `fd_read_lines()` uses a `NULL` sentinel to indicate that
* the array is at its end. As we use `free_names()` to free
* the array, we need to include this sentinel value here and
* thus have to allocate `readers_len + 1` many entries.
* thus have to allocate `tables_len + 1` many entries.
*/
REFTABLE_CALLOC_ARRAY(names, st->merged->readers_len + 1);
REFTABLE_CALLOC_ARRAY(names, st->merged->tables_len + 1);
if (!names) {
err = REFTABLE_OUT_OF_MEMORY_ERROR;
goto done;
}

for (size_t i = 0; i < st->merged->readers_len; i++) {
names[i] = reftable_strdup(st->readers[i]->name);
for (size_t i = 0; i < st->merged->tables_len; i++) {
names[i] = reftable_strdup(st->tables[i]->name);
if (!names[i]) {
err = REFTABLE_OUT_OF_MEMORY_ERROR;
goto done;
@ -1451,8 +1451,8 @@ static int stack_compact_range(struct reftable_stack *st,
* it into place now.
*/
if (!is_empty_table) {
err = format_name(&new_table_name, st->readers[first]->min_update_index,
st->readers[last]->max_update_index);
err = format_name(&new_table_name, st->tables[first]->min_update_index,
st->tables[last]->max_update_index);
if (err < 0)
goto done;

@ -1559,7 +1559,7 @@ done:
int reftable_stack_compact_all(struct reftable_stack *st,
struct reftable_log_expiry_config *config)
{
size_t last = st->merged->readers_len ? st->merged->readers_len - 1 : 0;
size_t last = st->merged->tables_len ? st->merged->tables_len - 1 : 0;
return stack_compact_range(st, 0, last, config, 0);
}

@ -1650,12 +1650,12 @@ static uint64_t *stack_table_sizes_for_compaction(struct reftable_stack *st)
int overhead = header_size(version) - 1;
uint64_t *sizes;

REFTABLE_CALLOC_ARRAY(sizes, st->merged->readers_len);
REFTABLE_CALLOC_ARRAY(sizes, st->merged->tables_len);
if (!sizes)
return NULL;

for (size_t i = 0; i < st->merged->readers_len; i++)
sizes[i] = st->readers[i]->size - overhead;
for (size_t i = 0; i < st->merged->tables_len; i++)
sizes[i] = st->tables[i]->size - overhead;

return sizes;
}
@ -1665,14 +1665,14 @@ int reftable_stack_auto_compact(struct reftable_stack *st)
struct segment seg;
uint64_t *sizes;

if (st->merged->readers_len < 2)
if (st->merged->tables_len < 2)
return 0;

sizes = stack_table_sizes_for_compaction(st);
if (!sizes)
return REFTABLE_OUT_OF_MEMORY_ERROR;

seg = suggest_compaction_segment(sizes, st->merged->readers_len,
seg = suggest_compaction_segment(sizes, st->merged->tables_len,
st->opts.auto_compaction_factor);
reftable_free(sizes);

@ -1763,7 +1763,7 @@ static void remove_maybe_stale_table(struct reftable_stack *st, uint64_t max,
int err = 0;
uint64_t update_idx = 0;
struct reftable_block_source src = { NULL };
struct reftable_reader *rd = NULL;
struct reftable_table *table = NULL;
struct reftable_buf table_path = REFTABLE_BUF_INIT;

err = stack_filename(&table_path, st, name);
@ -1774,12 +1774,12 @@ static void remove_maybe_stale_table(struct reftable_stack *st, uint64_t max,
if (err < 0)
goto done;

err = reftable_reader_new(&rd, &src, name);
err = reftable_table_new(&table, &src, name);
if (err < 0)
goto done;

update_idx = reftable_reader_max_update_index(rd);
reftable_reader_decref(rd);
update_idx = reftable_table_max_update_index(table);
reftable_table_decref(table);

if (update_idx <= max) {
unlink(table_path.buf);
@ -1803,8 +1803,8 @@ static int reftable_stack_clean_locked(struct reftable_stack *st)
if (!is_table_name(d->d_name))
continue;

for (size_t i = 0; !found && i < st->readers_len; i++)
found = !strcmp(reader_name(st->readers[i]), d->d_name);
for (size_t i = 0; !found && i < st->tables_len; i++)
found = !strcmp(reftable_table_name(st->tables[i]), d->d_name);
if (found)
continue;


16
reftable/stack.h
View File

@ -1,10 +1,10 @@
/*
Copyright 2020 Google LLC

Use of this source code is governed by a BSD-style
license that can be found in the LICENSE file or at
https://developers.google.com/open-source/licenses/bsd
*/
* Copyright 2020 Google LLC
*
* Use of this source code is governed by a BSD-style
* license that can be found in the LICENSE file or at
* https://developers.google.com/open-source/licenses/bsd
*/

#ifndef STACK_H
#define STACK_H
@ -22,8 +22,8 @@ struct reftable_stack {

struct reftable_write_options opts;

struct reftable_reader **readers;
size_t readers_len;
struct reftable_table **tables;
size_t tables_len;
struct reftable_merged_table *merged;
struct reftable_compaction_stats stats;
};

12
reftable/system.h
View File

@ -1,10 +1,10 @@
/*
Copyright 2020 Google LLC

Use of this source code is governed by a BSD-style
license that can be found in the LICENSE file or at
https://developers.google.com/open-source/licenses/bsd
*/
* Copyright 2020 Google LLC
*
* Use of this source code is governed by a BSD-style
* license that can be found in the LICENSE file or at
* https://developers.google.com/open-source/licenses/bsd
*/

#ifndef SYSTEM_H
#define SYSTEM_H

473
reftable/reader.c → reftable/table.c
View File

@ -1,86 +1,46 @@
/*
Copyright 2020 Google LLC
* Copyright 2020 Google LLC
*
* Use of this source code is governed by a BSD-style
* license that can be found in the LICENSE file or at
* https://developers.google.com/open-source/licenses/bsd
*/

Use of this source code is governed by a BSD-style
license that can be found in the LICENSE file or at
https://developers.google.com/open-source/licenses/bsd
*/

#include "reader.h"
#include "table.h"

#include "system.h"
#include "block.h"
#include "blocksource.h"
#include "constants.h"
#include "iter.h"
#include "record.h"
#include "reftable-error.h"

uint64_t block_source_size(struct reftable_block_source *source)
{
return source->ops->size(source->arg);
}

ssize_t block_source_read_block(struct reftable_block_source *source,
struct reftable_block *dest, uint64_t off,
uint32_t size)
{
ssize_t result = source->ops->read_block(source->arg, dest, off, size);
dest->source = *source;
return result;
}

void block_source_close(struct reftable_block_source *source)
{
if (!source->ops) {
return;
}

source->ops->close(source->arg);
source->ops = NULL;
}

static struct reftable_reader_offsets *
reader_offsets_for(struct reftable_reader *r, uint8_t typ)
static struct reftable_table_offsets *
table_offsets_for(struct reftable_table *t, uint8_t typ)
{
switch (typ) {
case BLOCK_TYPE_REF:
return &r->ref_offsets;
case BLOCK_TYPE_LOG:
return &r->log_offsets;
case BLOCK_TYPE_OBJ:
return &r->obj_offsets;
case REFTABLE_BLOCK_TYPE_REF:
return &t->ref_offsets;
case REFTABLE_BLOCK_TYPE_LOG:
return &t->log_offsets;
case REFTABLE_BLOCK_TYPE_OBJ:
return &t->obj_offsets;
}
abort();
}

static int reader_get_block(struct reftable_reader *r,
struct reftable_block *dest, uint64_t off,
uint32_t sz)
enum reftable_hash reftable_table_hash_id(struct reftable_table *t)
{
ssize_t bytes_read;
if (off >= r->size)
return 0;
if (off + sz > r->size)
sz = r->size - off;

bytes_read = block_source_read_block(&r->source, dest, off, sz);
if (bytes_read < 0)
return (int)bytes_read;

return 0;
return t->hash_id;
}

enum reftable_hash reftable_reader_hash_id(struct reftable_reader *r)
const char *reftable_table_name(struct reftable_table *t)
{
return r->hash_id;
return t->name;
}

const char *reader_name(struct reftable_reader *r)
{
return r->name;
}

static int parse_footer(struct reftable_reader *r, uint8_t *footer,
static int parse_footer(struct reftable_table *t, uint8_t *footer,
uint8_t *header)
{
uint8_t *f = footer;
@ -95,29 +55,29 @@ static int parse_footer(struct reftable_reader *r, uint8_t *footer,
}
f += 4;

if (memcmp(footer, header, header_size(r->version))) {
if (memcmp(footer, header, header_size(t->version))) {
err = REFTABLE_FORMAT_ERROR;
goto done;
}

f++;
r->block_size = reftable_get_be24(f);
t->block_size = reftable_get_be24(f);

f += 3;
r->min_update_index = reftable_get_be64(f);
t->min_update_index = reftable_get_be64(f);
f += 8;
r->max_update_index = reftable_get_be64(f);
t->max_update_index = reftable_get_be64(f);
f += 8;

if (r->version == 1) {
r->hash_id = REFTABLE_HASH_SHA1;
if (t->version == 1) {
t->hash_id = REFTABLE_HASH_SHA1;
} else {
switch (reftable_get_be32(f)) {
case REFTABLE_FORMAT_ID_SHA1:
r->hash_id = REFTABLE_HASH_SHA1;
t->hash_id = REFTABLE_HASH_SHA1;
break;
case REFTABLE_FORMAT_ID_SHA256:
r->hash_id = REFTABLE_HASH_SHA256;
t->hash_id = REFTABLE_HASH_SHA256;
break;
default:
err = REFTABLE_FORMAT_ERROR;
@ -127,20 +87,20 @@ static int parse_footer(struct reftable_reader *r, uint8_t *footer,
f += 4;
}

r->ref_offsets.index_offset = reftable_get_be64(f);
t->ref_offsets.index_offset = reftable_get_be64(f);
f += 8;

r->obj_offsets.offset = reftable_get_be64(f);
t->obj_offsets.offset = reftable_get_be64(f);
f += 8;

r->object_id_len = r->obj_offsets.offset & ((1 << 5) - 1);
r->obj_offsets.offset >>= 5;
t->object_id_len = t->obj_offsets.offset & ((1 << 5) - 1);
t->obj_offsets.offset >>= 5;

r->obj_offsets.index_offset = reftable_get_be64(f);
t->obj_offsets.index_offset = reftable_get_be64(f);
f += 8;
r->log_offsets.offset = reftable_get_be64(f);
t->log_offsets.offset = reftable_get_be64(f);
f += 8;
r->log_offsets.index_offset = reftable_get_be64(f);
t->log_offsets.index_offset = reftable_get_be64(f);
f += 8;

computed_crc = crc32(0, footer, f - footer);
@ -151,13 +111,13 @@ static int parse_footer(struct reftable_reader *r, uint8_t *footer,
goto done;
}

first_block_typ = header[header_size(r->version)];
r->ref_offsets.is_present = (first_block_typ == BLOCK_TYPE_REF);
r->ref_offsets.offset = 0;
r->log_offsets.is_present = (first_block_typ == BLOCK_TYPE_LOG ||
r->log_offsets.offset > 0);
r->obj_offsets.is_present = r->obj_offsets.offset > 0;
if (r->obj_offsets.is_present && !r->object_id_len) {
first_block_typ = header[header_size(t->version)];
t->ref_offsets.is_present = (first_block_typ == REFTABLE_BLOCK_TYPE_REF);
t->ref_offsets.offset = 0;
t->log_offsets.is_present = (first_block_typ == REFTABLE_BLOCK_TYPE_LOG ||
t->log_offsets.offset > 0);
t->obj_offsets.is_present = t->obj_offsets.offset > 0;
if (t->obj_offsets.is_present && !t->object_id_len) {
err = REFTABLE_FORMAT_ERROR;
goto done;
}
@ -168,20 +128,20 @@ done:
}

struct table_iter {
struct reftable_reader *r;
struct reftable_table *table;
uint8_t typ;
uint64_t block_off;
struct block_reader br;
struct reftable_block block;
struct block_iter bi;
int is_finished;
};

static int table_iter_init(struct table_iter *ti, struct reftable_reader *r)
static int table_iter_init(struct table_iter *ti, struct reftable_table *t)
{
struct block_iter bi = BLOCK_ITER_INIT;
memset(ti, 0, sizeof(*ti));
reftable_reader_incref(r);
ti->r = r;
reftable_table_incref(t);
ti->table = t;
ti->bi = bi;
return 0;
}
@ -190,8 +150,8 @@ static int table_iter_next_in_block(struct table_iter *ti,
struct reftable_record *rec)
{
int res = block_iter_next(&ti->bi, rec);
if (res == 0 && reftable_record_type(rec) == BLOCK_TYPE_REF) {
rec->u.ref.update_index += ti->r->min_update_index;
if (res == 0 && reftable_record_type(rec) == REFTABLE_BLOCK_TYPE_REF) {
rec->u.ref.update_index += ti->table->min_update_index;
}

return res;
@ -199,68 +159,32 @@ static int table_iter_next_in_block(struct table_iter *ti,

static void table_iter_block_done(struct table_iter *ti)
{
block_reader_release(&ti->br);
reftable_block_release(&ti->block);
block_iter_reset(&ti->bi);
}

static int32_t extract_block_size(uint8_t *data, uint8_t *typ, uint64_t off,
int version)
int table_init_block(struct reftable_table *t, struct reftable_block *block,
uint64_t next_off, uint8_t want_typ)
{
int32_t result = 0;
uint32_t header_off = next_off ? 0 : header_size(t->version);
int err;

if (off == 0) {
data += header_size(version);
}

*typ = data[0];
if (reftable_is_block_type(*typ)) {
result = reftable_get_be24(data + 1);
}
return result;
}

int reader_init_block_reader(struct reftable_reader *r, struct block_reader *br,
uint64_t next_off, uint8_t want_typ)
{
int32_t guess_block_size = r->block_size ? r->block_size :
DEFAULT_BLOCK_SIZE;
struct reftable_block block = { NULL };
uint8_t block_typ = 0;
int err = 0;
uint32_t header_off = next_off ? 0 : header_size(r->version);
int32_t block_size = 0;

if (next_off >= r->size)
if (next_off >= t->size)
return 1;

err = reader_get_block(r, &block, next_off, guess_block_size);
err = reftable_block_init(block, &t->source, next_off, header_off,
t->block_size, hash_size(t->hash_id));
if (err < 0)
goto done;

block_size = extract_block_size(block.data, &block_typ, next_off,
r->version);
if (block_size < 0) {
err = block_size;
goto done;
}
if (want_typ != BLOCK_TYPE_ANY && block_typ != want_typ) {
if (want_typ != REFTABLE_BLOCK_TYPE_ANY && block->block_type != want_typ) {
err = 1;
goto done;
}

if (block_size > guess_block_size) {
reftable_block_done(&block);
err = reader_get_block(r, &block, next_off, block_size);
if (err < 0) {
goto done;
}
}

err = block_reader_init(br, &block, header_off, r->block_size,
hash_size(r->hash_id));
done:
reftable_block_done(&block);

if (err)
reftable_block_release(block);
return err;
}

@ -268,15 +192,15 @@ static void table_iter_close(struct table_iter *ti)
{
table_iter_block_done(ti);
block_iter_close(&ti->bi);
reftable_reader_decref(ti->r);
reftable_table_decref(ti->table);
}

static int table_iter_next_block(struct table_iter *ti)
{
uint64_t next_block_off = ti->block_off + ti->br.full_block_size;
uint64_t next_block_off = ti->block_off + ti->block.full_block_size;
int err;

err = reader_init_block_reader(ti->r, &ti->br, next_block_off, ti->typ);
err = table_init_block(ti->table, &ti->block, next_block_off, ti->typ);
if (err > 0)
ti->is_finished = 1;
if (err)
@ -284,7 +208,7 @@ static int table_iter_next_block(struct table_iter *ti)

ti->block_off = next_block_off;
ti->is_finished = 0;
block_iter_seek_start(&ti->bi, &ti->br);
block_iter_init(&ti->bi, &ti->block);

return 0;
}
@ -326,27 +250,27 @@ static int table_iter_seek_to(struct table_iter *ti, uint64_t off, uint8_t typ)
{
int err;

err = reader_init_block_reader(ti->r, &ti->br, off, typ);
err = table_init_block(ti->table, &ti->block, off, typ);
if (err != 0)
return err;

ti->typ = block_reader_type(&ti->br);
ti->typ = reftable_block_type(&ti->block);
ti->block_off = off;
block_iter_seek_start(&ti->bi, &ti->br);
block_iter_init(&ti->bi, &ti->block);
ti->is_finished = 0;
return 0;
}

static int table_iter_seek_start(struct table_iter *ti, uint8_t typ, int index)
{
struct reftable_reader_offsets *offs = reader_offsets_for(ti->r, typ);
struct reftable_table_offsets *offs = table_offsets_for(ti->table, typ);
uint64_t off = offs->offset;
if (index) {
off = offs->index_offset;
if (off == 0) {
return 1;
}
typ = BLOCK_TYPE_INDEX;
typ = REFTABLE_BLOCK_TYPE_INDEX;
}

return table_iter_seek_to(ti, off, typ);
@ -396,10 +320,10 @@ static int table_iter_seek_linear(struct table_iter *ti,
* as we have more than three blocks we would have an index, so
* we would not do a linear search there anymore.
*/
memset(&next.br.block, 0, sizeof(next.br.block));
next.br.zstream = NULL;
next.br.uncompressed_data = NULL;
next.br.uncompressed_cap = 0;
memset(&next.block.block_data, 0, sizeof(next.block.block_data));
next.block.zstream = NULL;
next.block.uncompressed_data = NULL;
next.block.uncompressed_cap = 0;

err = table_iter_next_block(&next);
if (err < 0)
@ -407,7 +331,7 @@ static int table_iter_seek_linear(struct table_iter *ti,
if (err > 0)
break;

err = block_reader_first_key(&next.br, &got_key);
err = reftable_block_first_key(&next.block, &got_key);
if (err < 0)
goto done;

@ -425,7 +349,8 @@ static int table_iter_seek_linear(struct table_iter *ti,
* the wanted key inside of it. If the block does not contain our key
* we know that the corresponding record does not exist.
*/
err = block_iter_seek_key(&ti->bi, &ti->br, &want_key);
block_iter_init(&ti->bi, &ti->block);
err = block_iter_seek_key(&ti->bi, &want_key);
if (err < 0)
goto done;
err = 0;
@ -441,10 +366,10 @@ static int table_iter_seek_indexed(struct table_iter *ti,
struct reftable_record *rec)
{
struct reftable_record want_index = {
.type = BLOCK_TYPE_INDEX, .u.idx = { .last_key = REFTABLE_BUF_INIT }
.type = REFTABLE_BLOCK_TYPE_INDEX, .u.idx = { .last_key = REFTABLE_BUF_INIT }
};
struct reftable_record index_result = {
.type = BLOCK_TYPE_INDEX,
.type = REFTABLE_BLOCK_TYPE_INDEX,
.u.idx = { .last_key = REFTABLE_BUF_INIT },
};
int err;
@ -493,7 +418,9 @@ static int table_iter_seek_indexed(struct table_iter *ti,
if (err != 0)
goto done;

err = block_iter_seek_key(&ti->bi, &ti->br, &want_index.u.idx.last_key);
block_iter_init(&ti->bi, &ti->block);

err = block_iter_seek_key(&ti->bi, &want_index.u.idx.last_key);
if (err < 0)
goto done;

@ -502,7 +429,7 @@ static int table_iter_seek_indexed(struct table_iter *ti,
break;
}

if (ti->typ != BLOCK_TYPE_INDEX) {
if (ti->typ != REFTABLE_BLOCK_TYPE_INDEX) {
err = REFTABLE_FORMAT_ERROR;
goto done;
}
@ -518,7 +445,7 @@ static int table_iter_seek(struct table_iter *ti,
struct reftable_record *want)
{
uint8_t typ = reftable_record_type(want);
struct reftable_reader_offsets *offs = reader_offsets_for(ti->r, typ);
struct reftable_table_offsets *offs = table_offsets_for(ti->table, typ);
int err;

err = table_iter_seek_start(ti, reftable_record_type(want),
@ -566,11 +493,11 @@ static void iterator_from_table_iter(struct reftable_iterator *it,
it->ops = &table_iter_vtable;
}

int reader_init_iter(struct reftable_reader *r,
struct reftable_iterator *it,
uint8_t typ)
int table_init_iter(struct reftable_table *t,
struct reftable_iterator *it,
uint8_t typ)
{
struct reftable_reader_offsets *offs = reader_offsets_for(r, typ);
struct reftable_table_offsets *offs = table_offsets_for(t, typ);

if (offs->is_present) {
struct table_iter *ti;
@ -578,7 +505,7 @@ int reader_init_iter(struct reftable_reader *r,
if (!ti)
return REFTABLE_OUT_OF_MEMORY_ERROR;

table_iter_init(ti, r);
table_iter_init(ti, t);
iterator_from_table_iter(it, ti);
} else {
iterator_set_empty(it);
@ -587,31 +514,31 @@ int reader_init_iter(struct reftable_reader *r,
return 0;
}

int reftable_reader_init_ref_iterator(struct reftable_reader *r,
struct reftable_iterator *it)
int reftable_table_init_ref_iterator(struct reftable_table *t,
struct reftable_iterator *it)
{
return reader_init_iter(r, it, BLOCK_TYPE_REF);
return table_init_iter(t, it, REFTABLE_BLOCK_TYPE_REF);
}

int reftable_reader_init_log_iterator(struct reftable_reader *r,
struct reftable_iterator *it)
int reftable_table_init_log_iterator(struct reftable_table *t,
struct reftable_iterator *it)
{
return reader_init_iter(r, it, BLOCK_TYPE_LOG);
return table_init_iter(t, it, REFTABLE_BLOCK_TYPE_LOG);
}

int reftable_reader_new(struct reftable_reader **out,
struct reftable_block_source *source, char const *name)
int reftable_table_new(struct reftable_table **out,
struct reftable_block_source *source, char const *name)
{
struct reftable_block footer = { 0 };
struct reftable_block header = { 0 };
struct reftable_reader *r;
struct reftable_block_data footer = { 0 };
struct reftable_block_data header = { 0 };
struct reftable_table *t;
uint64_t file_size = block_source_size(source);
uint32_t read_size;
ssize_t bytes_read;
int err;

REFTABLE_CALLOC_ARRAY(r, 1);
if (!r) {
REFTABLE_CALLOC_ARRAY(t, 1);
if (!t) {
err = REFTABLE_OUT_OF_MEMORY_ERROR;
goto done;
}
@ -626,7 +553,7 @@ int reftable_reader_new(struct reftable_reader **out,
goto done;
}

bytes_read = block_source_read_block(source, &header, 0, read_size);
bytes_read = block_source_read_data(source, &header, 0, read_size);
if (bytes_read < 0 || (size_t)bytes_read != read_size) {
err = REFTABLE_IO_ERROR;
goto done;
@ -636,84 +563,84 @@ int reftable_reader_new(struct reftable_reader **out,
err = REFTABLE_FORMAT_ERROR;
goto done;
}
r->version = header.data[4];
if (r->version != 1 && r->version != 2) {
t->version = header.data[4];
if (t->version != 1 && t->version != 2) {
err = REFTABLE_FORMAT_ERROR;
goto done;
}

r->size = file_size - footer_size(r->version);
r->source = *source;
r->name = reftable_strdup(name);
if (!r->name) {
t->size = file_size - footer_size(t->version);
t->source = *source;
t->name = reftable_strdup(name);
if (!t->name) {
err = REFTABLE_OUT_OF_MEMORY_ERROR;
goto done;
}
r->hash_id = 0;
r->refcount = 1;
t->hash_id = 0;
t->refcount = 1;

bytes_read = block_source_read_block(source, &footer, r->size,
footer_size(r->version));
if (bytes_read < 0 || (size_t)bytes_read != footer_size(r->version)) {
bytes_read = block_source_read_data(source, &footer, t->size,
footer_size(t->version));
if (bytes_read < 0 || (size_t)bytes_read != footer_size(t->version)) {
err = REFTABLE_IO_ERROR;
goto done;
}

err = parse_footer(r, footer.data, header.data);
err = parse_footer(t, footer.data, header.data);
if (err)
goto done;

*out = r;
*out = t;

done:
reftable_block_done(&footer);
reftable_block_done(&header);
block_source_release_data(&footer);
block_source_release_data(&header);
if (err) {
if (r)
reftable_free(r->name);
reftable_free(r);
if (t)
reftable_free(t->name);
reftable_free(t);
block_source_close(source);
}
return err;
}

void reftable_reader_incref(struct reftable_reader *r)
void reftable_table_incref(struct reftable_table *t)
{
r->refcount++;
t->refcount++;
}

void reftable_reader_decref(struct reftable_reader *r)
void reftable_table_decref(struct reftable_table *t)
{
if (!r)
if (!t)
return;
if (--r->refcount)
if (--t->refcount)
return;
block_source_close(&r->source);
REFTABLE_FREE_AND_NULL(r->name);
reftable_free(r);
block_source_close(&t->source);
REFTABLE_FREE_AND_NULL(t->name);
reftable_free(t);
}

static int reftable_reader_refs_for_indexed(struct reftable_reader *r,
struct reftable_iterator *it,
uint8_t *oid)
static int reftable_table_refs_for_indexed(struct reftable_table *t,
struct reftable_iterator *it,
uint8_t *oid)
{
struct reftable_record want = {
.type = BLOCK_TYPE_OBJ,
.type = REFTABLE_BLOCK_TYPE_OBJ,
.u.obj = {
.hash_prefix = oid,
.hash_prefix_len = r->object_id_len,
.hash_prefix_len = t->object_id_len,
},
};
struct reftable_iterator oit = { NULL };
struct reftable_record got = {
.type = BLOCK_TYPE_OBJ,
.type = REFTABLE_BLOCK_TYPE_OBJ,
.u.obj = { 0 },
};
int err = 0;
struct indexed_table_ref_iter *itr = NULL;

/* Look through the reverse index. */
err = reader_init_iter(r, &oit, BLOCK_TYPE_OBJ);
err = table_init_iter(t, &oit, REFTABLE_BLOCK_TYPE_OBJ);
if (err < 0)
goto done;

@ -727,14 +654,14 @@ static int reftable_reader_refs_for_indexed(struct reftable_reader *r,
goto done;

if (err > 0 || memcmp(want.u.obj.hash_prefix, got.u.obj.hash_prefix,
r->object_id_len)) {
t->object_id_len)) {
/* didn't find it; return empty iterator */
iterator_set_empty(it);
err = 0;
goto done;
}

err = indexed_table_ref_iter_new(&itr, r, oid, hash_size(r->hash_id),
err = indexed_table_ref_iter_new(&itr, t, oid, hash_size(t->hash_id),
got.u.obj.offsets,
got.u.obj.offset_len);
if (err < 0)
@ -748,14 +675,14 @@ done:
return err;
}

static int reftable_reader_refs_for_unindexed(struct reftable_reader *r,
struct reftable_iterator *it,
uint8_t *oid)
static int reftable_table_refs_for_unindexed(struct reftable_table *t,
struct reftable_iterator *it,
uint8_t *oid)
{
struct table_iter *ti;
struct filtering_ref_iterator *filter = NULL;
struct filtering_ref_iterator empty = FILTERING_REF_ITERATOR_INIT;
uint32_t oid_len = hash_size(r->hash_id);
uint32_t oid_len = hash_size(t->hash_id);
int err;

REFTABLE_ALLOC_ARRAY(ti, 1);
@ -764,8 +691,8 @@ static int reftable_reader_refs_for_unindexed(struct reftable_reader *r,
goto out;
}

table_iter_init(ti, r);
err = table_iter_seek_start(ti, BLOCK_TYPE_REF, 0);
table_iter_init(ti, t);
err = table_iter_seek_start(ti, REFTABLE_BLOCK_TYPE_REF, 0);
if (err < 0)
goto out;

@ -795,85 +722,67 @@ out:
return err;
}

int reftable_reader_refs_for(struct reftable_reader *r,
struct reftable_iterator *it, uint8_t *oid)
int reftable_table_refs_for(struct reftable_table *t,
struct reftable_iterator *it, uint8_t *oid)
{
if (r->obj_offsets.is_present)
return reftable_reader_refs_for_indexed(r, it, oid);
return reftable_reader_refs_for_unindexed(r, it, oid);
if (t->obj_offsets.is_present)
return reftable_table_refs_for_indexed(t, it, oid);
return reftable_table_refs_for_unindexed(t, it, oid);
}

uint64_t reftable_reader_max_update_index(struct reftable_reader *r)
uint64_t reftable_table_max_update_index(struct reftable_table *t)
{
return r->max_update_index;
return t->max_update_index;
}

uint64_t reftable_reader_min_update_index(struct reftable_reader *r)
uint64_t reftable_table_min_update_index(struct reftable_table *t)
{
return r->min_update_index;
return t->min_update_index;
}

int reftable_reader_print_blocks(const char *tablename)
int reftable_table_iterator_init(struct reftable_table_iterator *it,
struct reftable_table *t)
{
struct {
const char *name;
int type;
} sections[] = {
{
.name = "ref",
.type = BLOCK_TYPE_REF,
},
{
.name = "obj",
.type = BLOCK_TYPE_OBJ,
},
{
.name = "log",
.type = BLOCK_TYPE_LOG,
},
};
struct reftable_block_source src = { 0 };
struct reftable_reader *r = NULL;
struct table_iter ti = { 0 };
size_t i;
struct table_iter *ti;
int err;

err = reftable_block_source_from_file(&src, tablename);
REFTABLE_ALLOC_ARRAY(ti, 1);
if (!ti)
return REFTABLE_OUT_OF_MEMORY_ERROR;

err = table_iter_init(ti, t);
if (err < 0)
goto done;
goto out;

err = reftable_reader_new(&r, &src, tablename);
it->iter_arg = ti;
err = 0;

out:
if (err < 0)
goto done;

table_iter_init(&ti, r);

printf("header:\n");
printf(" block_size: %d\n", r->block_size);

for (i = 0; i < sizeof(sections) / sizeof(*sections); i++) {
err = table_iter_seek_start(&ti, sections[i].type, 0);
if (err < 0)
goto done;
if (err > 0)
continue;

printf("%s:\n", sections[i].name);

while (1) {
printf(" - length: %u\n", ti.br.block_len);
printf(" restarts: %u\n", ti.br.restart_count);

err = table_iter_next_block(&ti);
if (err < 0)
goto done;
if (err > 0)
break;
}
}

done:
reftable_reader_decref(r);
table_iter_close(&ti);
reftable_free(ti);
return err;
}

void reftable_table_iterator_release(struct reftable_table_iterator *it)
{
if (!it->iter_arg)
return;
table_iter_close(it->iter_arg);
reftable_free(it->iter_arg);
it->iter_arg = NULL;
}

int reftable_table_iterator_next(struct reftable_table_iterator *it,
const struct reftable_block **out)
{
struct table_iter *ti = it->iter_arg;
int err;

err = table_iter_next_block(ti);
if (err)
return err;

*out = &ti->block;

return 0;
}

29
reftable/table.h Normal file
View File

@ -0,0 +1,29 @@
/*
* Copyright 2020 Google LLC
*
* Use of this source code is governed by a BSD-style
* license that can be found in the LICENSE file or at
* https://developers.google.com/open-source/licenses/bsd
*/

#ifndef TABLE_H
#define TABLE_H

#include "block.h"
#include "record.h"
#include "reftable-iterator.h"
#include "reftable-table.h"

const char *reftable_table_name(struct reftable_table *t);

int table_init_iter(struct reftable_table *t,
struct reftable_iterator *it,
uint8_t typ);

/*
* Initialize a block by reading from the given table and offset.
*/
int table_init_block(struct reftable_table *t, struct reftable_block *block,
uint64_t next_off, uint8_t want_typ);

#endif

12
reftable/tree.c
View File

@ -1,10 +1,10 @@
/*
Copyright 2020 Google LLC

Use of this source code is governed by a BSD-style
license that can be found in the LICENSE file or at
https://developers.google.com/open-source/licenses/bsd
*/
* Copyright 2020 Google LLC
*
* Use of this source code is governed by a BSD-style
* license that can be found in the LICENSE file or at
* https://developers.google.com/open-source/licenses/bsd
*/

#include "system.h"
#include "tree.h"

12
reftable/tree.h
View File

@ -1,10 +1,10 @@
/*
Copyright 2020 Google LLC

Use of this source code is governed by a BSD-style
license that can be found in the LICENSE file or at
https://developers.google.com/open-source/licenses/bsd
*/
* Copyright 2020 Google LLC
*
* Use of this source code is governed by a BSD-style
* license that can be found in the LICENSE file or at
* https://developers.google.com/open-source/licenses/bsd
*/

#ifndef TREE_H
#define TREE_H

34
reftable/writer.c
View File

@ -1,10 +1,10 @@
/*
Copyright 2020 Google LLC

Use of this source code is governed by a BSD-style
license that can be found in the LICENSE file or at
https://developers.google.com/open-source/licenses/bsd
*/
* Copyright 2020 Google LLC
*
* Use of this source code is governed by a BSD-style
* license that can be found in the LICENSE file or at
* https://developers.google.com/open-source/licenses/bsd
*/

#include "writer.h"

@ -172,7 +172,7 @@ int reftable_writer_new(struct reftable_writer **out,
wp->write_arg = writer_arg;
wp->opts = opts;
wp->flush = flush_func;
writer_reinit_block_writer(wp, BLOCK_TYPE_REF);
writer_reinit_block_writer(wp, REFTABLE_BLOCK_TYPE_REF);

*out = wp;

@ -342,7 +342,7 @@ int reftable_writer_add_ref(struct reftable_writer *w,
struct reftable_ref_record *ref)
{
struct reftable_record rec = {
.type = BLOCK_TYPE_REF,
.type = REFTABLE_BLOCK_TYPE_REF,
.u = {
.ref = *ref
},
@ -406,13 +406,13 @@ static int reftable_writer_add_log_verbatim(struct reftable_writer *w,
struct reftable_log_record *log)
{
struct reftable_record rec = {
.type = BLOCK_TYPE_LOG,
.type = REFTABLE_BLOCK_TYPE_LOG,
.u = {
.log = *log,
},
};
if (w->block_writer &&
block_writer_type(w->block_writer) == BLOCK_TYPE_REF) {
block_writer_type(w->block_writer) == REFTABLE_BLOCK_TYPE_REF) {
int err = writer_finish_public_section(w);
if (err < 0)
return err;
@ -532,7 +532,7 @@ static int writer_finish_section(struct reftable_writer *w)

max_level++;
index_start = w->next;
err = writer_reinit_block_writer(w, BLOCK_TYPE_INDEX);
err = writer_reinit_block_writer(w, REFTABLE_BLOCK_TYPE_INDEX);
if (err < 0)
return err;

@ -544,7 +544,7 @@ static int writer_finish_section(struct reftable_writer *w)
w->index_cap = 0;
for (i = 0; i < idx_len; i++) {
struct reftable_record rec = {
.type = BLOCK_TYPE_INDEX,
.type = REFTABLE_BLOCK_TYPE_INDEX,
.u = {
.idx = idx[i],
},
@ -609,7 +609,7 @@ static void write_object_record(void *void_arg, void *key)
struct write_record_arg *arg = void_arg;
struct obj_index_tree_node *entry = key;
struct reftable_record
rec = { .type = BLOCK_TYPE_OBJ,
rec = { .type = REFTABLE_BLOCK_TYPE_OBJ,
.u.obj = {
.hash_prefix = (uint8_t *)entry->hash.buf,
.hash_prefix_len = arg->w->stats.object_id_len,
@ -639,7 +639,7 @@ static void write_object_record(void *void_arg, void *key)
if (arg->err < 0)
goto done;

arg->err = writer_reinit_block_writer(arg->w, BLOCK_TYPE_OBJ);
arg->err = writer_reinit_block_writer(arg->w, REFTABLE_BLOCK_TYPE_OBJ);
if (arg->err < 0)
goto done;

@ -684,7 +684,7 @@ static int writer_dump_object_index(struct reftable_writer *w)
infix_walk(w->obj_index_tree, &update_common, &common);
w->stats.object_id_len = common.max + 1;

err = writer_reinit_block_writer(w, BLOCK_TYPE_OBJ);
err = writer_reinit_block_writer(w, REFTABLE_BLOCK_TYPE_OBJ);
if (err < 0)
return err;

@ -708,7 +708,7 @@ static int writer_finish_public_section(struct reftable_writer *w)
err = writer_finish_section(w);
if (err < 0)
return err;
if (typ == BLOCK_TYPE_REF && !w->opts.skip_index_objects &&
if (typ == REFTABLE_BLOCK_TYPE_REF && !w->opts.skip_index_objects &&
w->stats.ref_stats.index_blocks > 0) {
err = writer_dump_object_index(w);
if (err < 0)
@ -813,7 +813,7 @@ static int writer_flush_nonempty_block(struct reftable_writer *w)
* By default, all records except for log records are padded to the
* block size.
*/
if (!w->opts.unpadded && typ != BLOCK_TYPE_LOG)
if (!w->opts.unpadded && typ != REFTABLE_BLOCK_TYPE_LOG)
padding = w->opts.block_size - raw_bytes;

bstats = writer_reftable_block_stats(w, typ);

12
reftable/writer.h
View File

@ -1,10 +1,10 @@
/*
Copyright 2020 Google LLC

Use of this source code is governed by a BSD-style
license that can be found in the LICENSE file or at
https://developers.google.com/open-source/licenses/bsd
*/
* Copyright 2020 Google LLC
*
* Use of this source code is governed by a BSD-style
* license that can be found in the LICENSE file or at
* https://developers.google.com/open-source/licenses/bsd
*/

#ifndef WRITER_H
#define WRITER_H

81
t/helper/test-reftable.c
View File

@ -2,10 +2,11 @@
#include "hash.h"
#include "hex.h"
#include "reftable/system.h"
#include "reftable/reftable-constants.h"
#include "reftable/reftable-error.h"
#include "reftable/reftable-merged.h"
#include "reftable/reftable-reader.h"
#include "reftable/reftable-stack.h"
#include "reftable/reftable-table.h"
#include "test-tool.h"

static void print_help(void)
@ -20,6 +21,72 @@ static void print_help(void)
"\n");
}

static int dump_blocks(const char *tablename)
{
struct reftable_table_iterator ti = { 0 };
struct reftable_block_source src = { 0 };
struct reftable_table *table = NULL;
uint8_t section_type = 0;
int err;

err = reftable_block_source_from_file(&src, tablename);
if (err < 0)
goto done;

err = reftable_table_new(&table, &src, tablename);
if (err < 0)
goto done;

err = reftable_table_iterator_init(&ti, table);
if (err < 0)
goto done;

printf("header:\n");
printf(" block_size: %d\n", table->block_size);

while (1) {
const struct reftable_block *block;

err = reftable_table_iterator_next(&ti, &block);
if (err < 0)
goto done;
if (err > 0)
break;

if (block->block_type != section_type) {
const char *section;
switch (block->block_type) {
case REFTABLE_BLOCK_TYPE_LOG:
section = "log";
break;
case REFTABLE_BLOCK_TYPE_REF:
section = "ref";
break;
case REFTABLE_BLOCK_TYPE_OBJ:
section = "obj";
break;
case REFTABLE_BLOCK_TYPE_INDEX:
section = "idx";
break;
default:
err = -1;
goto done;
}

section_type = block->block_type;
printf("%s:\n", section);
}

printf(" - length: %u\n", block->restart_off);
printf(" restarts: %u\n", block->restart_count);
}

done:
reftable_table_iterator_release(&ti);
reftable_table_decref(table);
return err;
}

static int dump_table(struct reftable_merged_table *mt)
{
struct reftable_iterator it = { NULL };
@ -126,19 +193,19 @@ static int dump_reftable(const char *tablename)
{
struct reftable_block_source src = { 0 };
struct reftable_merged_table *mt = NULL;
struct reftable_reader *r = NULL;
struct reftable_table *table = NULL;
int err;

err = reftable_block_source_from_file(&src, tablename);
if (err < 0)
goto done;

err = reftable_reader_new(&r, &src, tablename);
err = reftable_table_new(&table, &src, tablename);
if (err < 0)
goto done;

err = reftable_merged_table_new(&mt, &r, 1,
reftable_reader_hash_id(r));
err = reftable_merged_table_new(&mt, &table, 1,
reftable_table_hash_id(table));
if (err < 0)
goto done;

@ -146,7 +213,7 @@ static int dump_reftable(const char *tablename)

done:
reftable_merged_table_free(mt);
reftable_reader_decref(r);
reftable_table_decref(table);
return err;
}

@ -184,7 +251,7 @@ int cmd__dump_reftable(int argc, const char **argv)
arg = argv[1];

if (opt_dump_blocks) {
err = reftable_reader_print_blocks(arg);
err = dump_blocks(arg);
} else if (opt_dump_table) {
err = dump_reftable(arg);
} else if (opt_dump_stack) {

2
t/meson.build
View File

@ -58,10 +58,10 @@ unit_test_programs = [
'unit-tests/t-reftable-block.c',
'unit-tests/t-reftable-merged.c',
'unit-tests/t-reftable-pq.c',
'unit-tests/t-reftable-reader.c',
'unit-tests/t-reftable-readwrite.c',
'unit-tests/t-reftable-record.c',
'unit-tests/t-reftable-stack.c',
'unit-tests/t-reftable-table.c',
]

foreach unit_test_program : unit_test_programs

9
t/t0613-reftable-write-options.sh
View File

@ -93,6 +93,9 @@ test_expect_success 'many refs results in multiple blocks' '
restarts: 3
- length: 3289
restarts: 3
idx:
- length: 103
restarts: 1
EOF
test-tool dump-reftable -b .git/reftable/*.ref >actual &&
test_cmp expect actual
@ -241,6 +244,9 @@ test_expect_success 'object index gets written by default with ref index' '
restarts: 1
- length: 80
restarts: 1
idx:
- length: 55
restarts: 2
obj:
- length: 11
restarts: 1
@ -277,6 +283,9 @@ test_expect_success 'object index can be disabled' '
restarts: 1
- length: 80
restarts: 1
idx:
- length: 55
restarts: 2
EOF
test-tool dump-reftable -b .git/reftable/*.ref >actual &&
test_cmp expect actual

214
t/unit-tests/t-reftable-block.c
View File

@ -19,24 +19,25 @@ static void t_ref_block_read_write(void)
struct reftable_record recs[30];
const size_t N = ARRAY_SIZE(recs);
const size_t block_size = 1024;
struct reftable_block block = { 0 };
struct reftable_block_source source = { 0 };
struct block_writer bw = {
.last_key = REFTABLE_BUF_INIT,
};
struct reftable_record rec = {
.type = BLOCK_TYPE_REF,
.type = REFTABLE_BLOCK_TYPE_REF,
};
size_t i = 0;
int ret;
struct block_reader br = { 0 };
struct reftable_block block = { 0 };
struct block_iter it = BLOCK_ITER_INIT;
struct reftable_buf want = REFTABLE_BUF_INIT, buf = REFTABLE_BUF_INIT;
struct reftable_buf want = REFTABLE_BUF_INIT;
struct reftable_buf block_data = REFTABLE_BUF_INIT;

REFTABLE_CALLOC_ARRAY(block.data, block_size);
check(block.data != NULL);
block.len = block_size;
block_source_from_buf(&block.source ,&buf);
ret = block_writer_init(&bw, BLOCK_TYPE_REF, block.data, block_size,
REFTABLE_CALLOC_ARRAY(block_data.buf, block_size);
check(block_data.buf != NULL);
block_data.len = block_size;

ret = block_writer_init(&bw, REFTABLE_BLOCK_TYPE_REF, (uint8_t *) block_data.buf, block_size,
header_off, hash_size(REFTABLE_HASH_SHA1));
check(!ret);

@ -62,9 +63,10 @@ static void t_ref_block_read_write(void)

block_writer_release(&bw);

block_reader_init(&br, &block, header_off, block_size, REFTABLE_HASH_SIZE_SHA1);
block_source_from_buf(&source ,&block_data);
reftable_block_init(&block, &source, 0, header_off, block_size, REFTABLE_HASH_SIZE_SHA1);

block_iter_seek_start(&it, &br);
block_iter_init(&it, &block);

for (i = 0; ; i++) {
ret = block_iter_next(&it, &rec);
@ -77,10 +79,9 @@ static void t_ref_block_read_write(void)
}

for (i = 0; i < N; i++) {
block_iter_reset(&it);
reftable_record_key(&recs[i], &want);

ret = block_iter_seek_key(&it, &br, &want);
ret = block_iter_seek_key(&it, &want);
check_int(ret, ==, 0);

ret = block_iter_next(&it, &rec);
@ -89,7 +90,7 @@ static void t_ref_block_read_write(void)
check(reftable_record_equal(&recs[i], &rec, REFTABLE_HASH_SIZE_SHA1));

want.len--;
ret = block_iter_seek_key(&it, &br, &want);
ret = block_iter_seek_key(&it, &want);
check_int(ret, ==, 0);

ret = block_iter_next(&it, &rec);
@ -97,12 +98,11 @@ static void t_ref_block_read_write(void)
check(reftable_record_equal(&recs[10 * (i / 10)], &rec, REFTABLE_HASH_SIZE_SHA1));
}

block_reader_release(&br);
reftable_block_release(&block);
block_iter_close(&it);
reftable_record_release(&rec);
reftable_block_done(&br.block);
reftable_buf_release(&want);
reftable_buf_release(&buf);
reftable_buf_release(&block_data);
for (i = 0; i < N; i++)
reftable_record_release(&recs[i]);
}
@ -113,24 +113,25 @@ static void t_log_block_read_write(void)
struct reftable_record recs[30];
const size_t N = ARRAY_SIZE(recs);
const size_t block_size = 2048;
struct reftable_block block = { 0 };
struct reftable_block_source source = { 0 };
struct block_writer bw = {
.last_key = REFTABLE_BUF_INIT,
};
struct reftable_record rec = {
.type = BLOCK_TYPE_LOG,
.type = REFTABLE_BLOCK_TYPE_LOG,
};
size_t i = 0;
int ret;
struct block_reader br = { 0 };
struct reftable_block block = { 0 };
struct block_iter it = BLOCK_ITER_INIT;
struct reftable_buf want = REFTABLE_BUF_INIT, buf = REFTABLE_BUF_INIT;
struct reftable_buf want = REFTABLE_BUF_INIT;
struct reftable_buf block_data = REFTABLE_BUF_INIT;

REFTABLE_CALLOC_ARRAY(block.data, block_size);
check(block.data != NULL);
block.len = block_size;
block_source_from_buf(&block.source ,&buf);
ret = block_writer_init(&bw, BLOCK_TYPE_LOG, block.data, block_size,
REFTABLE_CALLOC_ARRAY(block_data.buf, block_size);
check(block_data.buf != NULL);
block_data.len = block_size;

ret = block_writer_init(&bw, REFTABLE_BLOCK_TYPE_LOG, (uint8_t *) block_data.buf, block_size,
header_off, hash_size(REFTABLE_HASH_SHA1));
check(!ret);

@ -151,9 +152,10 @@ static void t_log_block_read_write(void)

block_writer_release(&bw);

block_reader_init(&br, &block, header_off, block_size, REFTABLE_HASH_SIZE_SHA1);
block_source_from_buf(&source, &block_data);
reftable_block_init(&block, &source, 0, header_off, block_size, REFTABLE_HASH_SIZE_SHA1);

block_iter_seek_start(&it, &br);
block_iter_init(&it, &block);

for (i = 0; ; i++) {
ret = block_iter_next(&it, &rec);
@ -166,11 +168,10 @@ static void t_log_block_read_write(void)
}

for (i = 0; i < N; i++) {
block_iter_reset(&it);
reftable_buf_reset(&want);
check(!reftable_buf_addstr(&want, recs[i].u.log.refname));

ret = block_iter_seek_key(&it, &br, &want);
ret = block_iter_seek_key(&it, &want);
check_int(ret, ==, 0);

ret = block_iter_next(&it, &rec);
@ -179,7 +180,7 @@ static void t_log_block_read_write(void)
check(reftable_record_equal(&recs[i], &rec, REFTABLE_HASH_SIZE_SHA1));

want.len--;
ret = block_iter_seek_key(&it, &br, &want);
ret = block_iter_seek_key(&it, &want);
check_int(ret, ==, 0);

ret = block_iter_next(&it, &rec);
@ -187,12 +188,11 @@ static void t_log_block_read_write(void)
check(reftable_record_equal(&recs[10 * (i / 10)], &rec, REFTABLE_HASH_SIZE_SHA1));
}

block_reader_release(&br);
reftable_block_release(&block);
block_iter_close(&it);
reftable_record_release(&rec);
reftable_block_done(&br.block);
reftable_buf_release(&want);
reftable_buf_release(&buf);
reftable_buf_release(&block_data);
for (i = 0; i < N; i++)
reftable_record_release(&recs[i]);
}
@ -203,24 +203,25 @@ static void t_obj_block_read_write(void)
struct reftable_record recs[30];
const size_t N = ARRAY_SIZE(recs);
const size_t block_size = 1024;
struct reftable_block block = { 0 };
struct reftable_block_source source = { 0 };
struct block_writer bw = {
.last_key = REFTABLE_BUF_INIT,
};
struct reftable_record rec = {
.type = BLOCK_TYPE_OBJ,
.type = REFTABLE_BLOCK_TYPE_OBJ,
};
size_t i = 0;
int ret;
struct block_reader br = { 0 };
struct reftable_block block = { 0 };
struct block_iter it = BLOCK_ITER_INIT;
struct reftable_buf want = REFTABLE_BUF_INIT, buf = REFTABLE_BUF_INIT;
struct reftable_buf want = REFTABLE_BUF_INIT;
struct reftable_buf block_data = REFTABLE_BUF_INIT;

REFTABLE_CALLOC_ARRAY(block.data, block_size);
check(block.data != NULL);
block.len = block_size;
block_source_from_buf(&block.source, &buf);
ret = block_writer_init(&bw, BLOCK_TYPE_OBJ, block.data, block_size,
REFTABLE_CALLOC_ARRAY(block_data.buf, block_size);
check(block_data.buf != NULL);
block_data.len = block_size;

ret = block_writer_init(&bw, REFTABLE_BLOCK_TYPE_OBJ, (uint8_t *) block_data.buf, block_size,
header_off, hash_size(REFTABLE_HASH_SHA1));
check(!ret);

@ -243,9 +244,10 @@ static void t_obj_block_read_write(void)

block_writer_release(&bw);

block_reader_init(&br, &block, header_off, block_size, REFTABLE_HASH_SIZE_SHA1);
block_source_from_buf(&source, &block_data);
reftable_block_init(&block, &source, 0, header_off, block_size, REFTABLE_HASH_SIZE_SHA1);

block_iter_seek_start(&it, &br);
block_iter_init(&it, &block);

for (i = 0; ; i++) {
ret = block_iter_next(&it, &rec);
@ -258,10 +260,9 @@ static void t_obj_block_read_write(void)
}

for (i = 0; i < N; i++) {
block_iter_reset(&it);
reftable_record_key(&recs[i], &want);

ret = block_iter_seek_key(&it, &br, &want);
ret = block_iter_seek_key(&it, &want);
check_int(ret, ==, 0);

ret = block_iter_next(&it, &rec);
@ -270,12 +271,11 @@ static void t_obj_block_read_write(void)
check(reftable_record_equal(&recs[i], &rec, REFTABLE_HASH_SIZE_SHA1));
}

block_reader_release(&br);
reftable_block_release(&block);
block_iter_close(&it);
reftable_record_release(&rec);
reftable_block_done(&br.block);
reftable_buf_release(&want);
reftable_buf_release(&buf);
reftable_buf_release(&block_data);
for (i = 0; i < N; i++)
reftable_record_release(&recs[i]);
}
@ -286,25 +286,26 @@ static void t_index_block_read_write(void)
struct reftable_record recs[30];
const size_t N = ARRAY_SIZE(recs);
const size_t block_size = 1024;
struct reftable_block block = { 0 };
struct reftable_block_source source = { 0 };
struct block_writer bw = {
.last_key = REFTABLE_BUF_INIT,
};
struct reftable_record rec = {
.type = BLOCK_TYPE_INDEX,
.type = REFTABLE_BLOCK_TYPE_INDEX,
.u.idx.last_key = REFTABLE_BUF_INIT,
};
size_t i = 0;
int ret;
struct block_reader br = { 0 };
struct reftable_block block = { 0 };
struct block_iter it = BLOCK_ITER_INIT;
struct reftable_buf want = REFTABLE_BUF_INIT, buf = REFTABLE_BUF_INIT;
struct reftable_buf want = REFTABLE_BUF_INIT;
struct reftable_buf block_data = REFTABLE_BUF_INIT;

REFTABLE_CALLOC_ARRAY(block.data, block_size);
check(block.data != NULL);
block.len = block_size;
block_source_from_buf(&block.source, &buf);
ret = block_writer_init(&bw, BLOCK_TYPE_INDEX, block.data, block_size,
REFTABLE_CALLOC_ARRAY(block_data.buf, block_size);
check(block_data.buf != NULL);
block_data.len = block_size;

ret = block_writer_init(&bw, REFTABLE_BLOCK_TYPE_INDEX, (uint8_t *) block_data.buf, block_size,
header_off, hash_size(REFTABLE_HASH_SHA1));
check(!ret);

@ -314,7 +315,7 @@ static void t_index_block_read_write(void)
snprintf(buf, sizeof(buf), "branch%02"PRIuMAX, (uintmax_t)i);

reftable_buf_init(&recs[i].u.idx.last_key);
recs[i].type = BLOCK_TYPE_INDEX;
recs[i].type = REFTABLE_BLOCK_TYPE_INDEX;
check(!reftable_buf_addstr(&recs[i].u.idx.last_key, buf));
recs[i].u.idx.offset = i;

@ -327,9 +328,10 @@ static void t_index_block_read_write(void)

block_writer_release(&bw);

block_reader_init(&br, &block, header_off, block_size, REFTABLE_HASH_SIZE_SHA1);
block_source_from_buf(&source, &block_data);
reftable_block_init(&block, &source, 0, header_off, block_size, REFTABLE_HASH_SIZE_SHA1);

block_iter_seek_start(&it, &br);
block_iter_init(&it, &block);

for (i = 0; ; i++) {
ret = block_iter_next(&it, &rec);
@ -342,10 +344,9 @@ static void t_index_block_read_write(void)
}

for (i = 0; i < N; i++) {
block_iter_reset(&it);
reftable_record_key(&recs[i], &want);

ret = block_iter_seek_key(&it, &br, &want);
ret = block_iter_seek_key(&it, &want);
check_int(ret, ==, 0);

ret = block_iter_next(&it, &rec);
@ -354,7 +355,7 @@ static void t_index_block_read_write(void)
check(reftable_record_equal(&recs[i], &rec, REFTABLE_HASH_SIZE_SHA1));

want.len--;
ret = block_iter_seek_key(&it, &br, &want);
ret = block_iter_seek_key(&it, &want);
check_int(ret, ==, 0);

ret = block_iter_next(&it, &rec);
@ -362,22 +363,99 @@ static void t_index_block_read_write(void)
check(reftable_record_equal(&recs[10 * (i / 10)], &rec, REFTABLE_HASH_SIZE_SHA1));
}

block_reader_release(&br);
reftable_block_release(&block);
block_iter_close(&it);
reftable_record_release(&rec);
reftable_block_done(&br.block);
reftable_buf_release(&want);
reftable_buf_release(&buf);
reftable_buf_release(&block_data);
for (i = 0; i < N; i++)
reftable_record_release(&recs[i]);
}

static void t_block_iterator(void)
{
struct reftable_block_source source = { 0 };
struct block_writer writer = {
.last_key = REFTABLE_BUF_INIT,
};
struct reftable_record expected_refs[20];
struct reftable_ref_record ref = { 0 };
struct reftable_iterator it = { 0 };
struct reftable_block block = { 0 };
struct reftable_buf data;
int err;

data.len = 1024;
REFTABLE_CALLOC_ARRAY(data.buf, data.len);
check(data.buf != NULL);

err = block_writer_init(&writer, REFTABLE_BLOCK_TYPE_REF, (uint8_t *) data.buf, data.len,
0, hash_size(REFTABLE_HASH_SHA1));
check(!err);

for (size_t i = 0; i < ARRAY_SIZE(expected_refs); i++) {
expected_refs[i] = (struct reftable_record) {
.type = REFTABLE_BLOCK_TYPE_REF,
.u.ref = {
.value_type = REFTABLE_REF_VAL1,
.refname = xstrfmt("refs/heads/branch-%02"PRIuMAX, (uintmax_t)i),
},
};
memset(expected_refs[i].u.ref.value.val1, i, REFTABLE_HASH_SIZE_SHA1);

err = block_writer_add(&writer, &expected_refs[i]);
check_int(err, ==, 0);
}

err = block_writer_finish(&writer);
check_int(err, >, 0);

block_source_from_buf(&source, &data);
reftable_block_init(&block, &source, 0, 0, data.len, REFTABLE_HASH_SIZE_SHA1);

err = reftable_block_init_iterator(&block, &it);
check_int(err, ==, 0);

for (size_t i = 0; ; i++) {
err = reftable_iterator_next_ref(&it, &ref);
if (err > 0) {
check_int(i, ==, ARRAY_SIZE(expected_refs));
break;
}
check_int(err, ==, 0);

check(reftable_ref_record_equal(&ref, &expected_refs[i].u.ref,
REFTABLE_HASH_SIZE_SHA1));
}

err = reftable_iterator_seek_ref(&it, "refs/heads/does-not-exist");
check_int(err, ==, 0);
err = reftable_iterator_next_ref(&it, &ref);
check_int(err, ==, 1);

err = reftable_iterator_seek_ref(&it, "refs/heads/branch-13");
check_int(err, ==, 0);
err = reftable_iterator_next_ref(&it, &ref);
check_int(err, ==, 0);
check(reftable_ref_record_equal(&ref, &expected_refs[13].u.ref,
REFTABLE_HASH_SIZE_SHA1));

for (size_t i = 0; i < ARRAY_SIZE(expected_refs); i++)
reftable_free(expected_refs[i].u.ref.refname);
reftable_ref_record_release(&ref);
reftable_iterator_destroy(&it);
reftable_block_release(&block);
block_writer_release(&writer);
reftable_buf_release(&data);
}

int cmd_main(int argc UNUSED, const char *argv[] UNUSED)
{
TEST(t_index_block_read_write(), "read-write operations on index blocks work");
TEST(t_log_block_read_write(), "read-write operations on log blocks work");
TEST(t_obj_block_read_write(), "read-write operations on obj blocks work");
TEST(t_ref_block_read_write(), "read-write operations on ref blocks work");
TEST(t_block_iterator(), "block iterator works");

return test_done();
}

86
t/unit-tests/t-reftable-merged.c
View File

@ -11,7 +11,7 @@ https://developers.google.com/open-source/licenses/bsd
#include "reftable/blocksource.h"
#include "reftable/constants.h"
#include "reftable/merged.h"
#include "reftable/reader.h"
#include "reftable/table.h"
#include "reftable/reftable-error.h"
#include "reftable/reftable-merged.h"
#include "reftable/reftable-writer.h"
@ -19,7 +19,7 @@ https://developers.google.com/open-source/licenses/bsd
static struct reftable_merged_table *
merged_table_from_records(struct reftable_ref_record **refs,
struct reftable_block_source **source,
struct reftable_reader ***readers, const size_t *sizes,
struct reftable_table ***tables, const size_t *sizes,
struct reftable_buf *buf, const size_t n)
{
struct reftable_merged_table *mt = NULL;
@ -28,8 +28,8 @@ merged_table_from_records(struct reftable_ref_record **refs,
};
int err;

REFTABLE_CALLOC_ARRAY(*readers, n);
check(*readers != NULL);
REFTABLE_CALLOC_ARRAY(*tables, n);
check(*tables != NULL);
REFTABLE_CALLOC_ARRAY(*source, n);
check(*source != NULL);

@ -37,21 +37,21 @@ merged_table_from_records(struct reftable_ref_record **refs,
t_reftable_write_to_buf(&buf[i], refs[i], sizes[i], NULL, 0, &opts);
block_source_from_buf(&(*source)[i], &buf[i]);

err = reftable_reader_new(&(*readers)[i], &(*source)[i],
"name");
err = reftable_table_new(&(*tables)[i], &(*source)[i],
"name");
check(!err);
}

err = reftable_merged_table_new(&mt, *readers, n, REFTABLE_HASH_SHA1);
err = reftable_merged_table_new(&mt, *tables, n, REFTABLE_HASH_SHA1);
check(!err);
return mt;
}

static void readers_destroy(struct reftable_reader **readers, const size_t n)
static void tables_destroy(struct reftable_table **tables, const size_t n)
{
for (size_t i = 0; i < n; i++)
reftable_reader_decref(readers[i]);
reftable_free(readers);
reftable_table_decref(tables[i]);
reftable_free(tables);
}

static void t_merged_single_record(void)
@ -77,14 +77,14 @@ static void t_merged_single_record(void)
size_t sizes[] = { ARRAY_SIZE(r1), ARRAY_SIZE(r2), ARRAY_SIZE(r3) };
struct reftable_buf bufs[3] = { REFTABLE_BUF_INIT, REFTABLE_BUF_INIT, REFTABLE_BUF_INIT };
struct reftable_block_source *bs = NULL;
struct reftable_reader **readers = NULL;
struct reftable_table **tables = NULL;
struct reftable_merged_table *mt =
merged_table_from_records(refs, &bs, &readers, sizes, bufs, 3);
merged_table_from_records(refs, &bs, &tables, sizes, bufs, 3);
struct reftable_ref_record ref = { 0 };
struct reftable_iterator it = { 0 };
int err;

err = merged_table_init_iter(mt, &it, BLOCK_TYPE_REF);
err = merged_table_init_iter(mt, &it, REFTABLE_BLOCK_TYPE_REF);
check(!err);
err = reftable_iterator_seek_ref(&it, "a");
check(!err);
@ -94,7 +94,7 @@ static void t_merged_single_record(void)
check(reftable_ref_record_equal(&r2[0], &ref, REFTABLE_HASH_SIZE_SHA1));
reftable_ref_record_release(&ref);
reftable_iterator_destroy(&it);
readers_destroy(readers, 3);
tables_destroy(tables, 3);
reftable_merged_table_free(mt);
for (size_t i = 0; i < ARRAY_SIZE(bufs); i++)
reftable_buf_release(&bufs[i]);
@ -154,9 +154,9 @@ static void t_merged_refs(void)
size_t sizes[3] = { ARRAY_SIZE(r1), ARRAY_SIZE(r2), ARRAY_SIZE(r3) };
struct reftable_buf bufs[3] = { REFTABLE_BUF_INIT, REFTABLE_BUF_INIT, REFTABLE_BUF_INIT };
struct reftable_block_source *bs = NULL;
struct reftable_reader **readers = NULL;
struct reftable_table **tables = NULL;
struct reftable_merged_table *mt =
merged_table_from_records(refs, &bs, &readers, sizes, bufs, 3);
merged_table_from_records(refs, &bs, &tables, sizes, bufs, 3);
struct reftable_iterator it = { 0 };
int err;
struct reftable_ref_record *out = NULL;
@ -164,7 +164,7 @@ static void t_merged_refs(void)
size_t cap = 0;
size_t i;

err = merged_table_init_iter(mt, &it, BLOCK_TYPE_REF);
err = merged_table_init_iter(mt, &it, REFTABLE_BLOCK_TYPE_REF);
check(!err);
err = reftable_iterator_seek_ref(&it, "a");
check(!err);
@ -193,7 +193,7 @@ static void t_merged_refs(void)

for (i = 0; i < 3; i++)
reftable_buf_release(&bufs[i]);
readers_destroy(readers, 3);
tables_destroy(tables, 3);
reftable_merged_table_free(mt);
reftable_free(bs);
}
@ -238,13 +238,13 @@ static void t_merged_seek_multiple_times(void)
REFTABLE_BUF_INIT, REFTABLE_BUF_INIT,
};
struct reftable_block_source *sources = NULL;
struct reftable_reader **readers = NULL;
struct reftable_table **tables = NULL;
struct reftable_ref_record rec = { 0 };
struct reftable_iterator it = { 0 };
struct reftable_merged_table *mt;

mt = merged_table_from_records(refs, &sources, &readers, sizes, bufs, 2);
merged_table_init_iter(mt, &it, BLOCK_TYPE_REF);
mt = merged_table_from_records(refs, &sources, &tables, sizes, bufs, 2);
merged_table_init_iter(mt, &it, REFTABLE_BLOCK_TYPE_REF);

for (size_t i = 0; i < 5; i++) {
int err = reftable_iterator_seek_ref(&it, "c");
@ -266,7 +266,7 @@ static void t_merged_seek_multiple_times(void)

for (size_t i = 0; i < ARRAY_SIZE(bufs); i++)
reftable_buf_release(&bufs[i]);
readers_destroy(readers, ARRAY_SIZE(refs));
tables_destroy(tables, ARRAY_SIZE(refs));
reftable_ref_record_release(&rec);
reftable_iterator_destroy(&it);
reftable_merged_table_free(mt);
@ -313,14 +313,14 @@ static void t_merged_seek_multiple_times_without_draining(void)
REFTABLE_BUF_INIT, REFTABLE_BUF_INIT,
};
struct reftable_block_source *sources = NULL;
struct reftable_reader **readers = NULL;
struct reftable_table **tables = NULL;
struct reftable_ref_record rec = { 0 };
struct reftable_iterator it = { 0 };
struct reftable_merged_table *mt;
int err;

mt = merged_table_from_records(refs, &sources, &readers, sizes, bufs, 2);
merged_table_init_iter(mt, &it, BLOCK_TYPE_REF);
mt = merged_table_from_records(refs, &sources, &tables, sizes, bufs, 2);
merged_table_init_iter(mt, &it, REFTABLE_BLOCK_TYPE_REF);

err = reftable_iterator_seek_ref(&it, "b");
check(!err);
@ -338,7 +338,7 @@ static void t_merged_seek_multiple_times_without_draining(void)

for (size_t i = 0; i < ARRAY_SIZE(bufs); i++)
reftable_buf_release(&bufs[i]);
readers_destroy(readers, ARRAY_SIZE(refs));
tables_destroy(tables, ARRAY_SIZE(refs));
reftable_ref_record_release(&rec);
reftable_iterator_destroy(&it);
reftable_merged_table_free(mt);
@ -348,7 +348,7 @@ static void t_merged_seek_multiple_times_without_draining(void)
static struct reftable_merged_table *
merged_table_from_log_records(struct reftable_log_record **logs,
struct reftable_block_source **source,
struct reftable_reader ***readers, const size_t *sizes,
struct reftable_table ***tables, const size_t *sizes,
struct reftable_buf *buf, const size_t n)
{
struct reftable_merged_table *mt = NULL;
@ -358,8 +358,8 @@ merged_table_from_log_records(struct reftable_log_record **logs,
};
int err;

REFTABLE_CALLOC_ARRAY(*readers, n);
check(*readers != NULL);
REFTABLE_CALLOC_ARRAY(*tables, n);
check(*tables != NULL);
REFTABLE_CALLOC_ARRAY(*source, n);
check(*source != NULL);

@ -367,12 +367,12 @@ merged_table_from_log_records(struct reftable_log_record **logs,
t_reftable_write_to_buf(&buf[i], NULL, 0, logs[i], sizes[i], &opts);
block_source_from_buf(&(*source)[i], &buf[i]);

err = reftable_reader_new(&(*readers)[i], &(*source)[i],
"name");
err = reftable_table_new(&(*tables)[i], &(*source)[i],
"name");
check(!err);
}

err = reftable_merged_table_new(&mt, *readers, n, REFTABLE_HASH_SHA1);
err = reftable_merged_table_new(&mt, *tables, n, REFTABLE_HASH_SHA1);
check(!err);
return mt;
}
@ -435,9 +435,9 @@ static void t_merged_logs(void)
size_t sizes[3] = { ARRAY_SIZE(r1), ARRAY_SIZE(r2), ARRAY_SIZE(r3) };
struct reftable_buf bufs[3] = { REFTABLE_BUF_INIT, REFTABLE_BUF_INIT, REFTABLE_BUF_INIT };
struct reftable_block_source *bs = NULL;
struct reftable_reader **readers = NULL;
struct reftable_table **tables = NULL;
struct reftable_merged_table *mt = merged_table_from_log_records(
logs, &bs, &readers, sizes, bufs, 3);
logs, &bs, &tables, sizes, bufs, 3);
struct reftable_iterator it = { 0 };
int err;
struct reftable_log_record *out = NULL;
@ -445,7 +445,7 @@ static void t_merged_logs(void)
size_t cap = 0;
size_t i;

err = merged_table_init_iter(mt, &it, BLOCK_TYPE_LOG);
err = merged_table_init_iter(mt, &it, REFTABLE_BLOCK_TYPE_LOG);
check(!err);
err = reftable_iterator_seek_log(&it, "a");
check(!err);
@ -469,7 +469,7 @@ static void t_merged_logs(void)
check(reftable_log_record_equal(want[i], &out[i],
REFTABLE_HASH_SIZE_SHA1));

err = merged_table_init_iter(mt, &it, BLOCK_TYPE_LOG);
err = merged_table_init_iter(mt, &it, REFTABLE_BLOCK_TYPE_LOG);
check(!err);
err = reftable_iterator_seek_log_at(&it, "a", 2);
check(!err);
@ -485,7 +485,7 @@ static void t_merged_logs(void)

for (i = 0; i < 3; i++)
reftable_buf_release(&bufs[i]);
readers_destroy(readers, 3);
tables_destroy(tables, 3);
reftable_merged_table_free(mt);
reftable_free(bs);
}
@ -502,7 +502,7 @@ static void t_default_write_opts(void)
int err;
struct reftable_block_source source = { 0 };
uint32_t hash_id;
struct reftable_reader *rd = NULL;
struct reftable_table *table = NULL;
struct reftable_merged_table *merged = NULL;

reftable_writer_set_limits(w, 1, 1);
@ -516,18 +516,18 @@ static void t_default_write_opts(void)

block_source_from_buf(&source, &buf);

err = reftable_reader_new(&rd, &source, "filename");
err = reftable_table_new(&table, &source, "filename");
check(!err);

hash_id = reftable_reader_hash_id(rd);
hash_id = reftable_table_hash_id(table);
check_int(hash_id, ==, REFTABLE_HASH_SHA1);

err = reftable_merged_table_new(&merged, &rd, 1, REFTABLE_HASH_SHA256);
err = reftable_merged_table_new(&merged, &table, 1, REFTABLE_HASH_SHA256);
check_int(err, ==, REFTABLE_FORMAT_ERROR);
err = reftable_merged_table_new(&merged, &rd, 1, REFTABLE_HASH_SHA1);
err = reftable_merged_table_new(&merged, &table, 1, REFTABLE_HASH_SHA1);
check(!err);

reftable_reader_decref(rd);
reftable_table_decref(table);
reftable_merged_table_free(merged);
reftable_buf_release(&buf);
}

10
t/unit-tests/t-reftable-pq.c
View File

@ -34,7 +34,7 @@ static void t_pq_record(void)
char *last = NULL;

for (i = 0; i < N; i++) {
check(!reftable_record_init(&recs[i], BLOCK_TYPE_REF));
check(!reftable_record_init(&recs[i], REFTABLE_BLOCK_TYPE_REF));
recs[i].u.ref.refname = xstrfmt("%02"PRIuMAX, (uintmax_t)i);
}

@ -57,7 +57,7 @@ static void t_pq_record(void)
merged_iter_pqueue_check(&pq);

check(pq_entry_equal(&top, &e));
check(reftable_record_type(e.rec) == BLOCK_TYPE_REF);
check(reftable_record_type(e.rec) == REFTABLE_BLOCK_TYPE_REF);
if (last)
check_int(strcmp(last, e.rec->u.ref.refname), <, 0);
last = e.rec->u.ref.refname;
@ -76,7 +76,7 @@ static void t_pq_index(void)
size_t N = ARRAY_SIZE(recs), i;

for (i = 0; i < N; i++) {
check(!reftable_record_init(&recs[i], BLOCK_TYPE_REF));
check(!reftable_record_init(&recs[i], REFTABLE_BLOCK_TYPE_REF));
recs[i].u.ref.refname = (char *) "refs/heads/master";
}

@ -100,7 +100,7 @@ static void t_pq_index(void)
merged_iter_pqueue_check(&pq);

check(pq_entry_equal(&top, &e));
check(reftable_record_type(e.rec) == BLOCK_TYPE_REF);
check(reftable_record_type(e.rec) == REFTABLE_BLOCK_TYPE_REF);
check_int(e.index, ==, i);
if (last)
check_str(last, e.rec->u.ref.refname);
@ -117,7 +117,7 @@ static void t_merged_iter_pqueue_top(void)
size_t N = ARRAY_SIZE(recs), i;

for (i = 0; i < N; i++) {
check(!reftable_record_init(&recs[i], BLOCK_TYPE_REF));
check(!reftable_record_init(&recs[i], REFTABLE_BLOCK_TYPE_REF));
recs[i].u.ref.refname = (char *) "refs/heads/master";
}


96
t/unit-tests/t-reftable-reader.c
View File

@ -1,96 +0,0 @@
#include "test-lib.h"
#include "lib-reftable.h"
#include "reftable/blocksource.h"
#include "reftable/reader.h"

static int t_reader_seek_once(void)
{
struct reftable_ref_record records[] = {
{
.refname = (char *) "refs/heads/main",
.value_type = REFTABLE_REF_VAL1,
.value.val1 = { 42 },
},
};
struct reftable_block_source source = { 0 };
struct reftable_ref_record ref = { 0 };
struct reftable_iterator it = { 0 };
struct reftable_reader *reader;
struct reftable_buf buf = REFTABLE_BUF_INIT;
int ret;

t_reftable_write_to_buf(&buf, records, ARRAY_SIZE(records), NULL, 0, NULL);
block_source_from_buf(&source, &buf);

ret = reftable_reader_new(&reader, &source, "name");
check(!ret);

reftable_reader_init_ref_iterator(reader, &it);
ret = reftable_iterator_seek_ref(&it, "");
check(!ret);
ret = reftable_iterator_next_ref(&it, &ref);
check(!ret);

ret = reftable_ref_record_equal(&ref, &records[0], REFTABLE_HASH_SIZE_SHA1);
check_int(ret, ==, 1);

ret = reftable_iterator_next_ref(&it, &ref);
check_int(ret, ==, 1);

reftable_ref_record_release(&ref);
reftable_iterator_destroy(&it);
reftable_reader_decref(reader);
reftable_buf_release(&buf);
return 0;
}

static int t_reader_reseek(void)
{
struct reftable_ref_record records[] = {
{
.refname = (char *) "refs/heads/main",
.value_type = REFTABLE_REF_VAL1,
.value.val1 = { 42 },
},
};
struct reftable_block_source source = { 0 };
struct reftable_ref_record ref = { 0 };
struct reftable_iterator it = { 0 };
struct reftable_reader *reader;
struct reftable_buf buf = REFTABLE_BUF_INIT;
int ret;

t_reftable_write_to_buf(&buf, records, ARRAY_SIZE(records), NULL, 0, NULL);
block_source_from_buf(&source, &buf);

ret = reftable_reader_new(&reader, &source, "name");
check(!ret);

reftable_reader_init_ref_iterator(reader, &it);

for (size_t i = 0; i < 5; i++) {
ret = reftable_iterator_seek_ref(&it, "");
check(!ret);
ret = reftable_iterator_next_ref(&it, &ref);
check(!ret);

ret = reftable_ref_record_equal(&ref, &records[0], REFTABLE_HASH_SIZE_SHA1);
check_int(ret, ==, 1);

ret = reftable_iterator_next_ref(&it, &ref);
check_int(ret, ==, 1);
}

reftable_ref_record_release(&ref);
reftable_iterator_destroy(&it);
reftable_reader_decref(reader);
reftable_buf_release(&buf);
return 0;
}

int cmd_main(int argc UNUSED, const char *argv[] UNUSED)
{
TEST(t_reader_seek_once(), "reader can seek once");
TEST(t_reader_reseek(), "reader can reseek multiple times");
return test_done();
}

106
t/unit-tests/t-reftable-readwrite.c
View File

@ -12,9 +12,9 @@ https://developers.google.com/open-source/licenses/bsd
#include "lib-reftable.h"
#include "reftable/basics.h"
#include "reftable/blocksource.h"
#include "reftable/reader.h"
#include "reftable/reftable-error.h"
#include "reftable/reftable-writer.h"
#include "reftable/table.h"
#include "strbuf.h"

static const int update_index = 5;
@ -23,22 +23,22 @@ static void t_buffer(void)
{
struct reftable_buf buf = REFTABLE_BUF_INIT;
struct reftable_block_source source = { 0 };
struct reftable_block out = { 0 };
struct reftable_block_data out = { 0 };
int n;
uint8_t in[] = "hello";
check(!reftable_buf_add(&buf, in, sizeof(in)));
block_source_from_buf(&source, &buf);
check_int(block_source_size(&source), ==, 6);
n = block_source_read_block(&source, &out, 0, sizeof(in));
n = block_source_read_data(&source, &out, 0, sizeof(in));
check_int(n, ==, sizeof(in));
check(!memcmp(in, out.data, n));
reftable_block_done(&out);
block_source_release_data(&out);

n = block_source_read_block(&source, &out, 1, 2);
n = block_source_read_data(&source, &out, 1, 2);
check_int(n, ==, 2);
check(!memcmp(out.data, "el", 2));

reftable_block_done(&out);
block_source_release_data(&out);
block_source_close(&source);
reftable_buf_release(&buf);
}
@ -204,7 +204,7 @@ static void t_log_write_read(void)
struct reftable_ref_record ref = { 0 };
struct reftable_log_record log = { 0 };
struct reftable_iterator it = { 0 };
struct reftable_reader *reader;
struct reftable_table *table;
struct reftable_block_source source = { 0 };
struct reftable_buf buf = REFTABLE_BUF_INIT;
struct reftable_writer *w = t_reftable_strbuf_writer(&buf, &opts);
@ -254,10 +254,10 @@ static void t_log_write_read(void)

block_source_from_buf(&source, &buf);

err = reftable_reader_new(&reader, &source, "file.log");
err = reftable_table_new(&table, &source, "file.log");
check(!err);

err = reftable_reader_init_ref_iterator(reader, &it);
err = reftable_table_init_ref_iterator(table, &it);
check(!err);

err = reftable_iterator_seek_ref(&it, names[N - 1]);
@ -273,7 +273,7 @@ static void t_log_write_read(void)
reftable_iterator_destroy(&it);
reftable_ref_record_release(&ref);

err = reftable_reader_init_log_iterator(reader, &it);
err = reftable_table_init_log_iterator(table, &it);
check(!err);
err = reftable_iterator_seek_log(&it, "");
check(!err);
@ -294,7 +294,7 @@ static void t_log_write_read(void)
/* cleanup. */
reftable_buf_release(&buf);
free_names(names);
reftable_reader_decref(reader);
reftable_table_decref(table);
}

static void t_log_zlib_corruption(void)
@ -303,7 +303,7 @@ static void t_log_zlib_corruption(void)
.block_size = 256,
};
struct reftable_iterator it = { 0 };
struct reftable_reader *reader;
struct reftable_table *table;
struct reftable_block_source source = { 0 };
struct reftable_buf buf = REFTABLE_BUF_INIT;
struct reftable_writer *w = t_reftable_strbuf_writer(&buf, &opts);
@ -345,10 +345,10 @@ static void t_log_zlib_corruption(void)

block_source_from_buf(&source, &buf);

err = reftable_reader_new(&reader, &source, "file.log");
err = reftable_table_new(&table, &source, "file.log");
check(!err);

err = reftable_reader_init_log_iterator(reader, &it);
err = reftable_table_init_log_iterator(table, &it);
check(!err);
err = reftable_iterator_seek_log(&it, "refname");
check_int(err, ==, REFTABLE_ZLIB_ERROR);
@ -356,7 +356,7 @@ static void t_log_zlib_corruption(void)
reftable_iterator_destroy(&it);

/* cleanup. */
reftable_reader_decref(reader);
reftable_table_decref(table);
reftable_buf_release(&buf);
}

@ -367,7 +367,7 @@ static void t_table_read_write_sequential(void)
int N = 50;
struct reftable_iterator it = { 0 };
struct reftable_block_source source = { 0 };
struct reftable_reader *reader;
struct reftable_table *table;
int err = 0;
int j = 0;

@ -375,10 +375,10 @@ static void t_table_read_write_sequential(void)

block_source_from_buf(&source, &buf);

err = reftable_reader_new(&reader, &source, "file.ref");
err = reftable_table_new(&table, &source, "file.ref");
check(!err);

err = reftable_reader_init_ref_iterator(reader, &it);
err = reftable_table_init_ref_iterator(table, &it);
check(!err);
err = reftable_iterator_seek_ref(&it, "");
check(!err);
@ -396,7 +396,7 @@ static void t_table_read_write_sequential(void)
check_int(j, ==, N);

reftable_iterator_destroy(&it);
reftable_reader_decref(reader);
reftable_table_decref(table);
reftable_buf_release(&buf);
free_names(names);
}
@ -417,7 +417,7 @@ static void t_table_read_api(void)
char **names;
struct reftable_buf buf = REFTABLE_BUF_INIT;
int N = 50;
struct reftable_reader *reader;
struct reftable_table *table;
struct reftable_block_source source = { 0 };
int err;
struct reftable_log_record log = { 0 };
@ -427,10 +427,10 @@ static void t_table_read_api(void)

block_source_from_buf(&source, &buf);

err = reftable_reader_new(&reader, &source, "file.ref");
err = reftable_table_new(&table, &source, "file.ref");
check(!err);

err = reftable_reader_init_ref_iterator(reader, &it);
err = reftable_table_init_ref_iterator(table, &it);
check(!err);
err = reftable_iterator_seek_ref(&it, names[0]);
check(!err);
@ -441,7 +441,7 @@ static void t_table_read_api(void)
reftable_buf_release(&buf);
free_names(names);
reftable_iterator_destroy(&it);
reftable_reader_decref(reader);
reftable_table_decref(table);
reftable_buf_release(&buf);
}

@ -450,7 +450,7 @@ static void t_table_read_write_seek(int index, enum reftable_hash hash_id)
char **names;
struct reftable_buf buf = REFTABLE_BUF_INIT;
int N = 50;
struct reftable_reader *reader;
struct reftable_table *table;
struct reftable_block_source source = { 0 };
int err;
int i = 0;
@ -463,18 +463,18 @@ static void t_table_read_write_seek(int index, enum reftable_hash hash_id)

block_source_from_buf(&source, &buf);

err = reftable_reader_new(&reader, &source, "file.ref");
err = reftable_table_new(&table, &source, "file.ref");
check(!err);
check_int(hash_id, ==, reftable_reader_hash_id(reader));
check_int(hash_id, ==, reftable_table_hash_id(table));

if (!index) {
reader->ref_offsets.index_offset = 0;
table->ref_offsets.index_offset = 0;
} else {
check_int(reader->ref_offsets.index_offset, >, 0);
check_int(table->ref_offsets.index_offset, >, 0);
}

for (i = 1; i < N; i++) {
err = reftable_reader_init_ref_iterator(reader, &it);
err = reftable_table_init_ref_iterator(table, &it);
check(!err);
err = reftable_iterator_seek_ref(&it, names[i]);
check(!err);
@ -491,7 +491,7 @@ static void t_table_read_write_seek(int index, enum reftable_hash hash_id)
check(!reftable_buf_addstr(&pastLast, names[N - 1]));
check(!reftable_buf_addstr(&pastLast, "/"));

err = reftable_reader_init_ref_iterator(reader, &it);
err = reftable_table_init_ref_iterator(table, &it);
check(!err);
err = reftable_iterator_seek_ref(&it, pastLast.buf);
if (err == 0) {
@ -507,7 +507,7 @@ static void t_table_read_write_seek(int index, enum reftable_hash hash_id)

reftable_buf_release(&buf);
free_names(names);
reftable_reader_decref(reader);
reftable_table_decref(table);
}

static void t_table_read_write_seek_linear(void)
@ -535,7 +535,7 @@ static void t_table_refs_for(int indexed)
.block_size = 256,
};
struct reftable_ref_record ref = { 0 };
struct reftable_reader *reader;
struct reftable_table *table;
struct reftable_block_source source = { 0 };
struct reftable_buf buf = REFTABLE_BUF_INIT;
struct reftable_writer *w = t_reftable_strbuf_writer(&buf, &opts);
@ -585,18 +585,18 @@ static void t_table_refs_for(int indexed)

block_source_from_buf(&source, &buf);

err = reftable_reader_new(&reader, &source, "file.ref");
err = reftable_table_new(&table, &source, "file.ref");
check(!err);
if (!indexed)
reader->obj_offsets.is_present = 0;
table->obj_offsets.is_present = 0;

err = reftable_reader_init_ref_iterator(reader, &it);
err = reftable_table_init_ref_iterator(table, &it);
check(!err);
err = reftable_iterator_seek_ref(&it, "");
check(!err);
reftable_iterator_destroy(&it);

err = reftable_reader_refs_for(reader, &it, want_hash);
err = reftable_table_refs_for(table, &it, want_hash);
check(!err);

for (j = 0; ; j++) {
@ -613,7 +613,7 @@ static void t_table_refs_for(int indexed)
reftable_buf_release(&buf);
free_names(want_names);
reftable_iterator_destroy(&it);
reftable_reader_decref(reader);
reftable_table_decref(table);
}

static void t_table_refs_for_no_index(void)
@ -632,7 +632,7 @@ static void t_write_empty_table(void)
struct reftable_buf buf = REFTABLE_BUF_INIT;
struct reftable_writer *w = t_reftable_strbuf_writer(&buf, &opts);
struct reftable_block_source source = { 0 };
struct reftable_reader *rd = NULL;
struct reftable_table *table = NULL;
struct reftable_ref_record rec = { 0 };
struct reftable_iterator it = { 0 };
int err;
@ -647,10 +647,10 @@ static void t_write_empty_table(void)

block_source_from_buf(&source, &buf);

err = reftable_reader_new(&rd, &source, "filename");
err = reftable_table_new(&table, &source, "filename");
check(!err);

err = reftable_reader_init_ref_iterator(rd, &it);
err = reftable_table_init_ref_iterator(table, &it);
check(!err);
err = reftable_iterator_seek_ref(&it, "");
check(!err);
@ -659,7 +659,7 @@ static void t_write_empty_table(void)
check_int(err, >, 0);

reftable_iterator_destroy(&it);
reftable_reader_decref(rd);
reftable_table_decref(table);
reftable_buf_release(&buf);
}

@ -803,7 +803,7 @@ static void t_write_multiple_indices(void)
struct reftable_iterator it = { 0 };
const struct reftable_stats *stats;
struct reftable_writer *writer;
struct reftable_reader *reader;
struct reftable_table *table;
char buf[128];
int err, i;

@ -852,21 +852,21 @@ static void t_write_multiple_indices(void)
check_int(stats->log_stats.index_offset, >, 0);

block_source_from_buf(&source, &writer_buf);
err = reftable_reader_new(&reader, &source, "filename");
err = reftable_table_new(&table, &source, "filename");
check(!err);

/*
* Seeking the log uses the log index now. In case there is any
* confusion regarding indices we would notice here.
*/
err = reftable_reader_init_log_iterator(reader, &it);
err = reftable_table_init_log_iterator(table, &it);
check(!err);
err = reftable_iterator_seek_log(&it, "");
check(!err);

reftable_iterator_destroy(&it);
reftable_writer_free(writer);
reftable_reader_decref(reader);
reftable_table_decref(table);
reftable_buf_release(&writer_buf);
}

@ -880,7 +880,7 @@ static void t_write_multi_level_index(void)
struct reftable_iterator it = { 0 };
const struct reftable_stats *stats;
struct reftable_writer *writer;
struct reftable_reader *reader;
struct reftable_table *table;
int err;

writer = t_reftable_strbuf_writer(&writer_buf, &opts);
@ -909,20 +909,20 @@ static void t_write_multi_level_index(void)
check_int(stats->ref_stats.max_index_level, ==, 2);

block_source_from_buf(&source, &writer_buf);
err = reftable_reader_new(&reader, &source, "filename");
err = reftable_table_new(&table, &source, "filename");
check(!err);

/*
* Seeking the last ref should work as expected.
*/
err = reftable_reader_init_ref_iterator(reader, &it);
err = reftable_table_init_ref_iterator(table, &it);
check(!err);
err = reftable_iterator_seek_ref(&it, "refs/heads/199");
check(!err);

reftable_iterator_destroy(&it);
reftable_writer_free(writer);
reftable_reader_decref(reader);
reftable_table_decref(table);
reftable_buf_release(&writer_buf);
reftable_buf_release(&buf);
}
@ -931,11 +931,11 @@ static void t_corrupt_table_empty(void)
{
struct reftable_buf buf = REFTABLE_BUF_INIT;
struct reftable_block_source source = { 0 };
struct reftable_reader *reader;
struct reftable_table *table;
int err;

block_source_from_buf(&source, &buf);
err = reftable_reader_new(&reader, &source, "file.log");
err = reftable_table_new(&table, &source, "file.log");
check_int(err, ==, REFTABLE_FORMAT_ERROR);
}

@ -944,12 +944,12 @@ static void t_corrupt_table(void)
uint8_t zeros[1024] = { 0 };
struct reftable_buf buf = REFTABLE_BUF_INIT;
struct reftable_block_source source = { 0 };
struct reftable_reader *reader;
struct reftable_table *table;
int err;
check(!reftable_buf_add(&buf, zeros, sizeof(zeros)));

block_source_from_buf(&source, &buf);
err = reftable_reader_new(&reader, &source, "file.log");
err = reftable_table_new(&table, &source, "file.log");
check_int(err, ==, REFTABLE_FORMAT_ERROR);

reftable_buf_release(&buf);

40
t/unit-tests/t-reftable-record.c
View File

@ -84,17 +84,17 @@ static void t_reftable_ref_record_comparison(void)
{
struct reftable_record in[3] = {
{
.type = BLOCK_TYPE_REF,
.type = REFTABLE_BLOCK_TYPE_REF,
.u.ref.refname = (char *) "refs/heads/master",
.u.ref.value_type = REFTABLE_REF_VAL1,
},
{
.type = BLOCK_TYPE_REF,
.type = REFTABLE_BLOCK_TYPE_REF,
.u.ref.refname = (char *) "refs/heads/master",
.u.ref.value_type = REFTABLE_REF_DELETION,
},
{
.type = BLOCK_TYPE_REF,
.type = REFTABLE_BLOCK_TYPE_REF,
.u.ref.refname = (char *) "HEAD",
.u.ref.value_type = REFTABLE_REF_SYMREF,
.u.ref.value.symref = (char *) "refs/heads/master",
@ -141,10 +141,10 @@ static void t_reftable_ref_record_roundtrip(void)

for (int i = REFTABLE_REF_DELETION; i < REFTABLE_NR_REF_VALUETYPES; i++) {
struct reftable_record in = {
.type = BLOCK_TYPE_REF,
.type = REFTABLE_BLOCK_TYPE_REF,
.u.ref.value_type = i,
};
struct reftable_record out = { .type = BLOCK_TYPE_REF };
struct reftable_record out = { .type = REFTABLE_BLOCK_TYPE_REF };
struct reftable_buf key = REFTABLE_BUF_INIT;
uint8_t buffer[1024] = { 0 };
struct string_view dest = {
@ -198,17 +198,17 @@ static void t_reftable_log_record_comparison(void)
{
struct reftable_record in[3] = {
{
.type = BLOCK_TYPE_LOG,
.type = REFTABLE_BLOCK_TYPE_LOG,
.u.log.refname = (char *) "refs/heads/master",
.u.log.update_index = 42,
},
{
.type = BLOCK_TYPE_LOG,
.type = REFTABLE_BLOCK_TYPE_LOG,
.u.log.refname = (char *) "refs/heads/master",
.u.log.update_index = 22,
},
{
.type = BLOCK_TYPE_LOG,
.type = REFTABLE_BLOCK_TYPE_LOG,
.u.log.refname = (char *) "refs/heads/main",
.u.log.update_index = 22,
},
@ -297,7 +297,7 @@ static void t_reftable_log_record_roundtrip(void)
check(!reftable_log_record_is_deletion(&in[2]));

for (size_t i = 0; i < ARRAY_SIZE(in); i++) {
struct reftable_record rec = { .type = BLOCK_TYPE_LOG };
struct reftable_record rec = { .type = REFTABLE_BLOCK_TYPE_LOG };
struct reftable_buf key = REFTABLE_BUF_INIT;
uint8_t buffer[1024] = { 0 };
struct string_view dest = {
@ -306,7 +306,7 @@ static void t_reftable_log_record_roundtrip(void)
};
/* populate out, to check for leaks. */
struct reftable_record out = {
.type = BLOCK_TYPE_LOG,
.type = REFTABLE_BLOCK_TYPE_LOG,
.u.log = {
.refname = xstrdup("old name"),
.value_type = REFTABLE_LOG_UPDATE,
@ -384,21 +384,21 @@ static void t_reftable_obj_record_comparison(void)
uint64_t offsets[] = { 0, 16, 32, 48, 64, 80, 96, 112};
struct reftable_record in[3] = {
{
.type = BLOCK_TYPE_OBJ,
.type = REFTABLE_BLOCK_TYPE_OBJ,
.u.obj.hash_prefix = id_bytes,
.u.obj.hash_prefix_len = 7,
.u.obj.offsets = offsets,
.u.obj.offset_len = 8,
},
{
.type = BLOCK_TYPE_OBJ,
.type = REFTABLE_BLOCK_TYPE_OBJ,
.u.obj.hash_prefix = id_bytes,
.u.obj.hash_prefix_len = 7,
.u.obj.offsets = offsets,
.u.obj.offset_len = 5,
},
{
.type = BLOCK_TYPE_OBJ,
.type = REFTABLE_BLOCK_TYPE_OBJ,
.u.obj.hash_prefix = id_bytes,
.u.obj.hash_prefix_len = 5,
},
@ -450,13 +450,13 @@ static void t_reftable_obj_record_roundtrip(void)
.len = sizeof(buffer),
};
struct reftable_record in = {
.type = BLOCK_TYPE_OBJ,
.type = REFTABLE_BLOCK_TYPE_OBJ,
.u = {
.obj = recs[i],
},
};
struct reftable_buf key = REFTABLE_BUF_INIT;
struct reftable_record out = { .type = BLOCK_TYPE_OBJ };
struct reftable_record out = { .type = REFTABLE_BLOCK_TYPE_OBJ };
int n, m;
uint8_t extra;

@ -482,17 +482,17 @@ static void t_reftable_index_record_comparison(void)
{
struct reftable_record in[3] = {
{
.type = BLOCK_TYPE_INDEX,
.type = REFTABLE_BLOCK_TYPE_INDEX,
.u.idx.offset = 22,
.u.idx.last_key = REFTABLE_BUF_INIT,
},
{
.type = BLOCK_TYPE_INDEX,
.type = REFTABLE_BLOCK_TYPE_INDEX,
.u.idx.offset = 32,
.u.idx.last_key = REFTABLE_BUF_INIT,
},
{
.type = BLOCK_TYPE_INDEX,
.type = REFTABLE_BLOCK_TYPE_INDEX,
.u.idx.offset = 32,
.u.idx.last_key = REFTABLE_BUF_INIT,
},
@ -523,7 +523,7 @@ static void t_reftable_index_record_comparison(void)
static void t_reftable_index_record_roundtrip(void)
{
struct reftable_record in = {
.type = BLOCK_TYPE_INDEX,
.type = REFTABLE_BLOCK_TYPE_INDEX,
.u.idx = {
.offset = 42,
.last_key = REFTABLE_BUF_INIT,
@ -537,7 +537,7 @@ static void t_reftable_index_record_roundtrip(void)
struct reftable_buf scratch = REFTABLE_BUF_INIT;
struct reftable_buf key = REFTABLE_BUF_INIT;
struct reftable_record out = {
.type = BLOCK_TYPE_INDEX,
.type = REFTABLE_BLOCK_TYPE_INDEX,
.u.idx = { .last_key = REFTABLE_BUF_INIT },
};
int n, m;

66
t/unit-tests/t-reftable-stack.c
View File

@ -12,9 +12,9 @@ https://developers.google.com/open-source/licenses/bsd
#include "lib-reftable.h"
#include "dir.h"
#include "reftable/merged.h"
#include "reftable/reader.h"
#include "reftable/reftable-error.h"
#include "reftable/stack.h"
#include "reftable/table.h"
#include "strbuf.h"
#include "tempfile.h"
#include <dirent.h>
@ -176,7 +176,7 @@ static void t_reftable_stack_add_one(void)
err = reftable_stack_read_ref(st, ref.refname, &dest);
check(!err);
check(reftable_ref_record_equal(&ref, &dest, REFTABLE_HASH_SIZE_SHA1));
check_int(st->readers_len, >, 0);
check_int(st->tables_len, >, 0);

#ifndef GIT_WINDOWS_NATIVE
check(!reftable_buf_addstr(&scratch, dir));
@ -189,7 +189,7 @@ static void t_reftable_stack_add_one(void)
check(!reftable_buf_addstr(&scratch, dir));
check(!reftable_buf_addstr(&scratch, "/"));
/* do not try at home; not an external API for reftable. */
check(!reftable_buf_addstr(&scratch, st->readers[0]->name));
check(!reftable_buf_addstr(&scratch, st->tables[0]->name));
err = stat(scratch.buf, &stat_result);
check(!err);
check_int((stat_result.st_mode & 0777), ==, opts.default_permissions);
@ -402,9 +402,9 @@ static void t_reftable_stack_transaction_api_performs_auto_compaction(void)
* all tables in the stack.
*/
if (i != n)
check_int(st->merged->readers_len, ==, i + 1);
check_int(st->merged->tables_len, ==, i + 1);
else
check_int(st->merged->readers_len, ==, 1);
check_int(st->merged->tables_len, ==, 1);
}

reftable_stack_destroy(st);
@ -430,7 +430,7 @@ static void t_reftable_stack_auto_compaction_fails_gracefully(void)

err = reftable_stack_add(st, write_test_ref, &ref);
check(!err);
check_int(st->merged->readers_len, ==, 1);
check_int(st->merged->tables_len, ==, 1);
check_int(st->stats.attempts, ==, 0);
check_int(st->stats.failures, ==, 0);

@ -441,14 +441,14 @@ static void t_reftable_stack_auto_compaction_fails_gracefully(void)
*/
check(!reftable_buf_addstr(&table_path, dir));
check(!reftable_buf_addstr(&table_path, "/"));
check(!reftable_buf_addstr(&table_path, st->readers[0]->name));
check(!reftable_buf_addstr(&table_path, st->tables[0]->name));
check(!reftable_buf_addstr(&table_path, ".lock"));
write_file_buf(table_path.buf, "", 0);

ref.update_index = 2;
err = reftable_stack_add(st, write_test_ref, &ref);
check(!err);
check_int(st->merged->readers_len, ==, 2);
check_int(st->merged->tables_len, ==, 2);
check_int(st->stats.attempts, ==, 1);
check_int(st->stats.failures, ==, 1);

@ -592,7 +592,7 @@ static void t_reftable_stack_add(void)
check(!reftable_buf_addstr(&path, dir));
check(!reftable_buf_addstr(&path, "/"));
/* do not try at home; not an external API for reftable. */
check(!reftable_buf_addstr(&path, st->readers[0]->name));
check(!reftable_buf_addstr(&path, st->tables[0]->name));
err = stat(path.buf, &stat_result);
check(!err);
check_int((stat_result.st_mode & 0777), ==, opts.default_permissions);
@ -1026,7 +1026,7 @@ static void t_reftable_stack_auto_compaction(void)

err = reftable_stack_auto_compact(st);
check(!err);
check(i < 2 || st->merged->readers_len < 2 * fastlogN(i, 2));
check(i < 2 || st->merged->tables_len < 2 * fastlogN(i, 2));
}

check_int(reftable_stack_compaction_stats(st)->entries_written, <,
@ -1061,7 +1061,7 @@ static void t_reftable_stack_auto_compaction_factor(void)
err = reftable_stack_add(st, &write_test_ref, &ref);
check(!err);

check(i < 5 || st->merged->readers_len < 5 * fastlogN(i, 5));
check(i < 5 || st->merged->tables_len < 5 * fastlogN(i, 5));
}

reftable_stack_destroy(st);
@ -1082,7 +1082,7 @@ static void t_reftable_stack_auto_compaction_with_locked_tables(void)
check(!err);

write_n_ref_tables(st, 5);
check_int(st->merged->readers_len, ==, 5);
check_int(st->merged->tables_len, ==, 5);

/*
* Given that all tables we have written should be roughly the same
@ -1091,7 +1091,7 @@ static void t_reftable_stack_auto_compaction_with_locked_tables(void)
*/
check(!reftable_buf_addstr(&buf, dir));
check(!reftable_buf_addstr(&buf, "/"));
check(!reftable_buf_addstr(&buf, st->readers[2]->name));
check(!reftable_buf_addstr(&buf, st->tables[2]->name));
check(!reftable_buf_addstr(&buf, ".lock"));
write_file_buf(buf.buf, "", 0);

@ -1104,7 +1104,7 @@ static void t_reftable_stack_auto_compaction_with_locked_tables(void)
err = reftable_stack_auto_compact(st);
check(!err);
check_int(st->stats.failures, ==, 0);
check_int(st->merged->readers_len, ==, 4);
check_int(st->merged->tables_len, ==, 4);

reftable_stack_destroy(st);
reftable_buf_release(&buf);
@ -1149,9 +1149,9 @@ static void t_reftable_stack_add_performs_auto_compaction(void)
* all tables in the stack.
*/
if (i != n)
check_int(st->merged->readers_len, ==, i + 1);
check_int(st->merged->tables_len, ==, i + 1);
else
check_int(st->merged->readers_len, ==, 1);
check_int(st->merged->tables_len, ==, 1);
}

reftable_stack_destroy(st);
@ -1172,12 +1172,12 @@ static void t_reftable_stack_compaction_with_locked_tables(void)
check(!err);

write_n_ref_tables(st, 3);
check_int(st->merged->readers_len, ==, 3);
check_int(st->merged->tables_len, ==, 3);

/* Lock one of the tables that we're about to compact. */
check(!reftable_buf_addstr(&buf, dir));
check(!reftable_buf_addstr(&buf, "/"));
check(!reftable_buf_addstr(&buf, st->readers[1]->name));
check(!reftable_buf_addstr(&buf, st->tables[1]->name));
check(!reftable_buf_addstr(&buf, ".lock"));
write_file_buf(buf.buf, "", 0);

@ -1188,7 +1188,7 @@ static void t_reftable_stack_compaction_with_locked_tables(void)
err = reftable_stack_compact_all(st, NULL);
check_int(err, ==, REFTABLE_LOCK_ERROR);
check_int(st->stats.failures, ==, 1);
check_int(st->merged->readers_len, ==, 3);
check_int(st->merged->tables_len, ==, 3);

reftable_stack_destroy(st);
reftable_buf_release(&buf);
@ -1222,10 +1222,10 @@ static void t_reftable_stack_compaction_concurrent(void)
static void unclean_stack_close(struct reftable_stack *st)
{
/* break abstraction boundary to simulate unclean shutdown. */
for (size_t i = 0; i < st->readers_len; i++)
reftable_reader_decref(st->readers[i]);
st->readers_len = 0;
REFTABLE_FREE_AND_NULL(st->readers);
for (size_t i = 0; i < st->tables_len; i++)
reftable_table_decref(st->tables[i]);
st->tables_len = 0;
REFTABLE_FREE_AND_NULL(st->tables);
}

static void t_reftable_stack_compaction_concurrent_clean(void)
@ -1275,7 +1275,7 @@ static void t_reftable_stack_read_across_reload(void)
err = reftable_new_stack(&st1, dir, &opts);
check(!err);
write_n_ref_tables(st1, 2);
check_int(st1->merged->readers_len, ==, 2);
check_int(st1->merged->tables_len, ==, 2);
reftable_stack_init_ref_iterator(st1, &it);
err = reftable_iterator_seek_ref(&it, "");
check(!err);
@ -1283,10 +1283,10 @@ static void t_reftable_stack_read_across_reload(void)
/* Set up a second stack for the same directory and compact it. */
err = reftable_new_stack(&st2, dir, &opts);
check(!err);
check_int(st2->merged->readers_len, ==, 2);
check_int(st2->merged->tables_len, ==, 2);
err = reftable_stack_compact_all(st2, NULL);
check(!err);
check_int(st2->merged->readers_len, ==, 1);
check_int(st2->merged->tables_len, ==, 1);

/*
* Verify that we can continue to use the old iterator even after we
@ -1294,7 +1294,7 @@ static void t_reftable_stack_read_across_reload(void)
*/
err = reftable_stack_reload(st1);
check(!err);
check_int(st1->merged->readers_len, ==, 1);
check_int(st1->merged->tables_len, ==, 1);
err = reftable_iterator_next_ref(&it, &rec);
check(!err);
check_str(rec.refname, "refs/heads/branch-0000");
@ -1325,19 +1325,19 @@ static void t_reftable_stack_reload_with_missing_table(void)
err = reftable_new_stack(&st, dir, &opts);
check(!err);
write_n_ref_tables(st, 2);
check_int(st->merged->readers_len, ==, 2);
check_int(st->merged->tables_len, ==, 2);
reftable_stack_init_ref_iterator(st, &it);
err = reftable_iterator_seek_ref(&it, "");
check(!err);

/*
* Update the tables.list file with some garbage data, while reusing
* our old readers. This should trigger a partial reload of the stack,
* where we try to reuse our old readers.
* our old tables. This should trigger a partial reload of the stack,
* where we try to reuse our old tables.
*/
check(!reftable_buf_addstr(&content, st->readers[0]->name));
check(!reftable_buf_addstr(&content, st->tables[0]->name));
check(!reftable_buf_addstr(&content, "\n"));
check(!reftable_buf_addstr(&content, st->readers[1]->name));
check(!reftable_buf_addstr(&content, st->tables[1]->name));
check(!reftable_buf_addstr(&content, "\n"));
check(!reftable_buf_addstr(&content, "garbage\n"));
check(!reftable_buf_addstr(&table_path, st->list_file));
@ -1348,7 +1348,7 @@ static void t_reftable_stack_reload_with_missing_table(void)

err = reftable_stack_reload(st);
check_int(err, ==, -4);
check_int(st->merged->readers_len, ==, 2);
check_int(st->merged->tables_len, ==, 2);

/*
* Even though the reload has failed, we should be able to continue

206
t/unit-tests/t-reftable-table.c Normal file
View File

@ -0,0 +1,206 @@
#include "test-lib.h"
#include "lib-reftable.h"
#include "reftable/blocksource.h"
#include "reftable/constants.h"
#include "reftable/iter.h"
#include "reftable/table.h"
#include "strbuf.h"

static int t_table_seek_once(void)
{
struct reftable_ref_record records[] = {
{
.refname = (char *) "refs/heads/main",
.value_type = REFTABLE_REF_VAL1,
.value.val1 = { 42 },
},
};
struct reftable_block_source source = { 0 };
struct reftable_ref_record ref = { 0 };
struct reftable_iterator it = { 0 };
struct reftable_table *table;
struct reftable_buf buf = REFTABLE_BUF_INIT;
int ret;

t_reftable_write_to_buf(&buf, records, ARRAY_SIZE(records), NULL, 0, NULL);
block_source_from_buf(&source, &buf);

ret = reftable_table_new(&table, &source, "name");
check(!ret);

reftable_table_init_ref_iterator(table, &it);
ret = reftable_iterator_seek_ref(&it, "");
check(!ret);
ret = reftable_iterator_next_ref(&it, &ref);
check(!ret);

ret = reftable_ref_record_equal(&ref, &records[0], REFTABLE_HASH_SIZE_SHA1);
check_int(ret, ==, 1);

ret = reftable_iterator_next_ref(&it, &ref);
check_int(ret, ==, 1);

reftable_ref_record_release(&ref);
reftable_iterator_destroy(&it);
reftable_table_decref(table);
reftable_buf_release(&buf);
return 0;
}

static int t_table_reseek(void)
{
struct reftable_ref_record records[] = {
{
.refname = (char *) "refs/heads/main",
.value_type = REFTABLE_REF_VAL1,
.value.val1 = { 42 },
},
};
struct reftable_block_source source = { 0 };
struct reftable_ref_record ref = { 0 };
struct reftable_iterator it = { 0 };
struct reftable_table *table;
struct reftable_buf buf = REFTABLE_BUF_INIT;
int ret;

t_reftable_write_to_buf(&buf, records, ARRAY_SIZE(records), NULL, 0, NULL);
block_source_from_buf(&source, &buf);

ret = reftable_table_new(&table, &source, "name");
check(!ret);

reftable_table_init_ref_iterator(table, &it);

for (size_t i = 0; i < 5; i++) {
ret = reftable_iterator_seek_ref(&it, "");
check(!ret);
ret = reftable_iterator_next_ref(&it, &ref);
check(!ret);

ret = reftable_ref_record_equal(&ref, &records[0], REFTABLE_HASH_SIZE_SHA1);
check_int(ret, ==, 1);

ret = reftable_iterator_next_ref(&it, &ref);
check_int(ret, ==, 1);
}

reftable_ref_record_release(&ref);
reftable_iterator_destroy(&it);
reftable_table_decref(table);
reftable_buf_release(&buf);
return 0;
}

static int t_table_block_iterator(void)
{
struct reftable_block_source source = { 0 };
struct reftable_table_iterator it = { 0 };
struct reftable_ref_record *records;
const struct reftable_block *block;
struct reftable_table *table;
struct reftable_buf buf = REFTABLE_BUF_INIT;
struct {
uint8_t block_type;
uint16_t header_off;
uint16_t restart_count;
uint16_t record_count;
} expected_blocks[] = {
{
.block_type = REFTABLE_BLOCK_TYPE_REF,
.header_off = 24,
.restart_count = 10,
.record_count = 158,
},
{
.block_type = REFTABLE_BLOCK_TYPE_REF,
.restart_count = 10,
.record_count = 159,
},
{
.block_type = REFTABLE_BLOCK_TYPE_REF,
.restart_count = 10,
.record_count = 159,
},
{
.block_type = REFTABLE_BLOCK_TYPE_REF,
.restart_count = 2,
.record_count = 24,
},
{
.block_type = REFTABLE_BLOCK_TYPE_INDEX,
.restart_count = 1,
.record_count = 4,
},
{
.block_type = REFTABLE_BLOCK_TYPE_OBJ,
.restart_count = 1,
.record_count = 1,
},
};
const size_t nrecords = 500;
int ret;

REFTABLE_CALLOC_ARRAY(records, nrecords);
for (size_t i = 0; i < nrecords; i++) {
records[i].value_type = REFTABLE_REF_VAL1;
records[i].refname = xstrfmt("refs/heads/branch-%03"PRIuMAX,
(uintmax_t) i);
}

t_reftable_write_to_buf(&buf, records, nrecords, NULL, 0, NULL);
block_source_from_buf(&source, &buf);

ret = reftable_table_new(&table, &source, "name");
check(!ret);

ret = reftable_table_iterator_init(&it, table);
check(!ret);

for (size_t i = 0; i < ARRAY_SIZE(expected_blocks); i++) {
struct reftable_iterator record_it = { 0 };
struct reftable_record record = {
.type = expected_blocks[i].block_type,
};

ret = reftable_table_iterator_next(&it, &block);
check(!ret);

check_int(block->block_type, ==, expected_blocks[i].block_type);
check_int(block->header_off, ==, expected_blocks[i].header_off);
check_int(block->restart_count, ==, expected_blocks[i].restart_count);

ret = reftable_block_init_iterator(block, &record_it);
check(!ret);

for (size_t j = 0; ; j++) {
ret = iterator_next(&record_it, &record);
if (ret > 0) {
check_int(j, ==, expected_blocks[i].record_count);
break;
}
check(!ret);
}

reftable_iterator_destroy(&record_it);
reftable_record_release(&record);
}

ret = reftable_table_iterator_next(&it, &block);
check_int(ret, ==, 1);

for (size_t i = 0; i < nrecords; i++)
reftable_free(records[i].refname);
reftable_table_iterator_release(&it);
reftable_table_decref(table);
reftable_buf_release(&buf);
reftable_free(records);
return 0;
}

int cmd_main(int argc UNUSED, const char *argv[] UNUSED)
{
TEST(t_table_seek_once(), "table can seek once");
TEST(t_table_reseek(), "table can reseek multiple times");
TEST(t_table_block_iterator(), "table can iterate through blocks");
return test_done();
}