Merge branch 'ps/odb-move-loose-object-writing' into next
The logic to write loose objects has been refactored and moved from 'object-file.c' to the loose backend source file 'odb/source-loose.c', making the loose backend more self-contained. This is achieved by first refactoring force_object_loose() to use generic ODB write interfaces instead of loose-backend internals. * ps/odb-move-loose-object-writing: object-file: move logic to write loose objects object-file: move `force_object_loose()` object-file: force objects loose via generic interface object-file: fix memory leak in `force_object_loose()` odb: support setting mtime when writing objects odb: lift object existence check out of the "loose" backend odb: compute object hash in `odb_write_object_ext()` t/u-odb-inmemory: implement wrapper for writing objects odb: compute compat object ID in `odb_write_object_ext()`next
commit
244d577d93
|
|
@ -32,6 +32,7 @@
|
|||
#include "list.h"
|
||||
#include "packfile.h"
|
||||
#include "object-file.h"
|
||||
#include "object-file-convert.h"
|
||||
#include "odb.h"
|
||||
#include "odb/streaming.h"
|
||||
#include "replace-object.h"
|
||||
|
|
@ -4621,6 +4622,51 @@ static int loosened_object_can_be_discarded(const struct object_id *oid,
|
|||
return 1;
|
||||
}
|
||||
|
||||
static int force_object_loose(struct odb_source *source,
|
||||
const struct object_id *oid,
|
||||
const time_t *mtime)
|
||||
{
|
||||
struct odb_source_files *files = odb_source_files_downcast(source);
|
||||
const struct git_hash_algo *compat = source->odb->repo->compat_hash_algo;
|
||||
struct object_info oi = OBJECT_INFO_INIT;
|
||||
struct object_id compat_oid, *compat_oid_p = NULL;
|
||||
enum object_type type;
|
||||
void *buf = NULL;
|
||||
size_t len;
|
||||
int ret;
|
||||
|
||||
for (struct odb_source *s = source->odb->sources; s; s = s->next) {
|
||||
struct odb_source_files *files = odb_source_files_downcast(s);
|
||||
if (!odb_source_read_object_info(&files->loose->base, oid, NULL, 0))
|
||||
return 0;
|
||||
}
|
||||
|
||||
oi.typep = &type;
|
||||
oi.sizep = &len;
|
||||
oi.contentp = &buf;
|
||||
if (odb_read_object_info_extended(source->odb, oid, &oi, 0)) {
|
||||
ret = error(_("cannot read object for %s"), oid_to_hex(oid));
|
||||
goto out;
|
||||
}
|
||||
|
||||
if (compat) {
|
||||
if (repo_oid_to_algop(source->odb->repo, oid, compat, &compat_oid)) {
|
||||
ret = error(_("cannot map object %s to %s"),
|
||||
oid_to_hex(oid), compat->name);
|
||||
goto out;
|
||||
}
|
||||
|
||||
compat_oid_p = &compat_oid;
|
||||
}
|
||||
|
||||
ret = odb_source_write_object(&files->loose->base, buf, len, type, oid,
|
||||
compat_oid_p, mtime, 0);
|
||||
|
||||
out:
|
||||
free(buf);
|
||||
return ret;
|
||||
}
|
||||
|
||||
static void loosen_unused_packed_objects(void)
|
||||
{
|
||||
struct packed_git *p;
|
||||
|
|
@ -4641,7 +4687,7 @@ static void loosen_unused_packed_objects(void)
|
|||
!has_sha1_pack_kept_or_nonlocal(&oid) &&
|
||||
!loosened_object_can_be_discarded(&oid, p->mtime)) {
|
||||
if (force_object_loose(the_repository->objects->sources,
|
||||
&oid, p->mtime))
|
||||
&oid, &p->mtime))
|
||||
die(_("unable to force loose object"));
|
||||
loosened_objects_nr++;
|
||||
}
|
||||
|
|
|
|||
449
object-file.c
449
object-file.c
|
|
@ -67,9 +67,17 @@ const char *odb_loose_path(struct odb_source_loose *loose,
|
|||
}
|
||||
|
||||
/* Returns 1 if we have successfully freshened the file, 0 otherwise. */
|
||||
static int freshen_file(const char *fn)
|
||||
static int freshen_file(const char *fn, const time_t *mtime)
|
||||
{
|
||||
return !utime(fn, NULL);
|
||||
struct utimbuf times, *timesp = NULL;
|
||||
|
||||
if (mtime) {
|
||||
times.actime = *mtime;
|
||||
times.modtime = *mtime;
|
||||
timesp = ×
|
||||
}
|
||||
|
||||
return !utime(fn, timesp);
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
@ -79,11 +87,12 @@ static int freshen_file(const char *fn)
|
|||
* either does not exist on disk, or has a stale mtime and may be subject to
|
||||
* pruning).
|
||||
*/
|
||||
int check_and_freshen_file(const char *fn, int freshen)
|
||||
int check_and_freshen_file(const char *fn, int freshen,
|
||||
const time_t *mtime)
|
||||
{
|
||||
if (access(fn, F_OK))
|
||||
return 0;
|
||||
if (freshen && !freshen_file(fn))
|
||||
if (freshen && !freshen_file(fn, mtime))
|
||||
return 0;
|
||||
return 1;
|
||||
}
|
||||
|
|
@ -313,31 +322,6 @@ int parse_loose_header(const char *hdr, struct object_info *oi)
|
|||
return 0;
|
||||
}
|
||||
|
||||
static void hash_object_body(const struct git_hash_algo *algo, struct git_hash_ctx *c,
|
||||
const void *buf, size_t len,
|
||||
struct object_id *oid,
|
||||
char *hdr, size_t *hdrlen)
|
||||
{
|
||||
git_hash_init(c, algo);
|
||||
git_hash_update(c, hdr, *hdrlen);
|
||||
git_hash_update(c, buf, len);
|
||||
git_hash_final_oid(oid, c);
|
||||
}
|
||||
|
||||
void write_object_file_prepare(const struct git_hash_algo *algo,
|
||||
const void *buf, size_t len,
|
||||
enum object_type type, struct object_id *oid,
|
||||
char *hdr, size_t *hdrlen)
|
||||
{
|
||||
struct git_hash_ctx c;
|
||||
|
||||
/* Generate the header */
|
||||
*hdrlen = format_object_header(hdr, *hdrlen, type, len);
|
||||
|
||||
/* Hash (function pointers) computation */
|
||||
hash_object_body(algo, &c, buf, len, oid, hdr, hdrlen);
|
||||
}
|
||||
|
||||
#define CHECK_COLLISION_DEST_VANISHED -2
|
||||
|
||||
static int check_collision(const char *source, const char *dest)
|
||||
|
|
@ -474,10 +458,16 @@ void hash_object_file(const struct git_hash_algo *algo, const void *buf,
|
|||
size_t len, enum object_type type,
|
||||
struct object_id *oid)
|
||||
{
|
||||
struct git_hash_ctx c;
|
||||
char hdr[MAX_HEADER_LEN];
|
||||
size_t hdrlen = sizeof(hdr);
|
||||
int hdrlen;
|
||||
|
||||
write_object_file_prepare(algo, buf, len, type, oid, hdr, &hdrlen);
|
||||
hdrlen = format_object_header(hdr, sizeof(hdr), type, len);
|
||||
|
||||
git_hash_init(&c, algo);
|
||||
git_hash_update(&c, hdr, hdrlen);
|
||||
git_hash_update(&c, buf, len);
|
||||
git_hash_final_oid(oid, &c);
|
||||
}
|
||||
|
||||
struct transaction_packfile {
|
||||
|
|
@ -499,7 +489,7 @@ struct odb_transaction_files {
|
|||
const char *prefix;
|
||||
};
|
||||
|
||||
static int odb_transaction_files_prepare(struct odb_transaction *base)
|
||||
int odb_transaction_files_prepare(struct odb_transaction *base)
|
||||
{
|
||||
struct odb_transaction_files *transaction =
|
||||
container_of_or_null(base, struct odb_transaction_files, base);
|
||||
|
|
@ -522,8 +512,8 @@ static int odb_transaction_files_prepare(struct odb_transaction *base)
|
|||
return 0;
|
||||
}
|
||||
|
||||
static void odb_transaction_files_fsync(struct odb_transaction *base,
|
||||
int fd, const char *filename)
|
||||
void odb_transaction_files_fsync(struct odb_transaction *base,
|
||||
int fd, const char *filename)
|
||||
{
|
||||
struct odb_transaction_files *transaction =
|
||||
container_of_or_null(base, struct odb_transaction_files, base);
|
||||
|
|
@ -547,397 +537,6 @@ static void odb_transaction_files_fsync(struct odb_transaction *base,
|
|||
}
|
||||
}
|
||||
|
||||
/* Finalize a file on disk, and close it. */
|
||||
static void close_loose_object(struct odb_source_loose *loose,
|
||||
int fd, const char *filename)
|
||||
{
|
||||
if (loose->base.will_destroy)
|
||||
goto out;
|
||||
|
||||
if (batch_fsync_enabled(FSYNC_COMPONENT_LOOSE_OBJECT))
|
||||
odb_transaction_files_fsync(loose->base.odb->transaction, fd, filename);
|
||||
else if (fsync_object_files > 0)
|
||||
fsync_or_die(fd, filename);
|
||||
else
|
||||
fsync_component_or_die(FSYNC_COMPONENT_LOOSE_OBJECT, fd,
|
||||
filename);
|
||||
|
||||
out:
|
||||
if (close(fd) != 0)
|
||||
die_errno(_("error when closing loose object file"));
|
||||
}
|
||||
|
||||
/* Size of directory component, including the ending '/' */
|
||||
static inline int directory_size(const char *filename)
|
||||
{
|
||||
const char *s = strrchr(filename, '/');
|
||||
if (!s)
|
||||
return 0;
|
||||
return s - filename + 1;
|
||||
}
|
||||
|
||||
/*
|
||||
* This creates a temporary file in the same directory as the final
|
||||
* 'filename'
|
||||
*
|
||||
* We want to avoid cross-directory filename renames, because those
|
||||
* can have problems on various filesystems (FAT, NFS, Coda).
|
||||
*/
|
||||
static int create_tmpfile(struct repository *repo,
|
||||
struct strbuf *tmp, const char *filename)
|
||||
{
|
||||
int fd, dirlen = directory_size(filename);
|
||||
|
||||
strbuf_reset(tmp);
|
||||
strbuf_add(tmp, filename, dirlen);
|
||||
strbuf_addstr(tmp, "tmp_obj_XXXXXX");
|
||||
fd = git_mkstemp_mode(tmp->buf, 0444);
|
||||
if (fd < 0 && dirlen && errno == ENOENT) {
|
||||
/*
|
||||
* Make sure the directory exists; note that the contents
|
||||
* of the buffer are undefined after mkstemp returns an
|
||||
* error, so we have to rewrite the whole buffer from
|
||||
* scratch.
|
||||
*/
|
||||
strbuf_reset(tmp);
|
||||
strbuf_add(tmp, filename, dirlen - 1);
|
||||
if (mkdir(tmp->buf, 0777) && errno != EEXIST)
|
||||
return -1;
|
||||
if (adjust_shared_perm(repo, tmp->buf))
|
||||
return -1;
|
||||
|
||||
/* Try again */
|
||||
strbuf_addstr(tmp, "/tmp_obj_XXXXXX");
|
||||
fd = git_mkstemp_mode(tmp->buf, 0444);
|
||||
}
|
||||
return fd;
|
||||
}
|
||||
|
||||
/**
|
||||
* Common steps for loose object writers to start writing loose
|
||||
* objects:
|
||||
*
|
||||
* - Create tmpfile for the loose object.
|
||||
* - Setup zlib stream for compression.
|
||||
* - Start to feed header to zlib stream.
|
||||
*
|
||||
* Returns a "fd", which should later be provided to
|
||||
* end_loose_object_common().
|
||||
*/
|
||||
static int start_loose_object_common(struct odb_source_loose *loose,
|
||||
struct strbuf *tmp_file,
|
||||
const char *filename, unsigned flags,
|
||||
git_zstream *stream,
|
||||
unsigned char *buf, size_t buflen,
|
||||
struct git_hash_ctx *c, struct git_hash_ctx *compat_c,
|
||||
char *hdr, int hdrlen)
|
||||
{
|
||||
const struct git_hash_algo *algo = loose->base.odb->repo->hash_algo;
|
||||
const struct git_hash_algo *compat = loose->base.odb->repo->compat_hash_algo;
|
||||
int fd;
|
||||
struct repo_config_values *cfg = repo_config_values(the_repository);
|
||||
|
||||
fd = create_tmpfile(loose->base.odb->repo, tmp_file, filename);
|
||||
if (fd < 0) {
|
||||
if (flags & ODB_WRITE_OBJECT_SILENT)
|
||||
return -1;
|
||||
else if (errno == EACCES)
|
||||
return error(_("insufficient permission for adding "
|
||||
"an object to repository database %s"),
|
||||
loose->base.path);
|
||||
else
|
||||
return error_errno(
|
||||
_("unable to create temporary file"));
|
||||
}
|
||||
|
||||
/* Setup zlib stream for compression */
|
||||
git_deflate_init(stream, cfg->zlib_compression_level);
|
||||
stream->next_out = buf;
|
||||
stream->avail_out = buflen;
|
||||
git_hash_init(c, algo);
|
||||
if (compat && compat_c)
|
||||
git_hash_init(compat_c, compat);
|
||||
|
||||
/* Start to feed header to zlib stream */
|
||||
stream->next_in = (unsigned char *)hdr;
|
||||
stream->avail_in = hdrlen;
|
||||
while (git_deflate(stream, 0) == Z_OK)
|
||||
; /* nothing */
|
||||
git_hash_update(c, hdr, hdrlen);
|
||||
if (compat && compat_c)
|
||||
git_hash_update(compat_c, hdr, hdrlen);
|
||||
|
||||
return fd;
|
||||
}
|
||||
|
||||
/**
|
||||
* Common steps for the inner git_deflate() loop for writing loose
|
||||
* objects. Returns what git_deflate() returns.
|
||||
*/
|
||||
static int write_loose_object_common(struct odb_source_loose *loose,
|
||||
struct git_hash_ctx *c, struct git_hash_ctx *compat_c,
|
||||
git_zstream *stream, const int flush,
|
||||
unsigned char *in0, const int fd,
|
||||
unsigned char *compressed,
|
||||
const size_t compressed_len)
|
||||
{
|
||||
const struct git_hash_algo *compat = loose->base.odb->repo->compat_hash_algo;
|
||||
int ret;
|
||||
|
||||
ret = git_deflate(stream, flush ? Z_FINISH : 0);
|
||||
git_hash_update(c, in0, stream->next_in - in0);
|
||||
if (compat && compat_c)
|
||||
git_hash_update(compat_c, in0, stream->next_in - in0);
|
||||
if (write_in_full(fd, compressed, stream->next_out - compressed) < 0)
|
||||
die_errno(_("unable to write loose object file"));
|
||||
stream->next_out = compressed;
|
||||
stream->avail_out = compressed_len;
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
/**
|
||||
* Common steps for loose object writers to end writing loose objects:
|
||||
*
|
||||
* - End the compression of zlib stream.
|
||||
* - Get the calculated oid to "oid".
|
||||
*/
|
||||
static int end_loose_object_common(struct odb_source_loose *loose,
|
||||
struct git_hash_ctx *c, struct git_hash_ctx *compat_c,
|
||||
git_zstream *stream, struct object_id *oid,
|
||||
struct object_id *compat_oid)
|
||||
{
|
||||
const struct git_hash_algo *compat = loose->base.odb->repo->compat_hash_algo;
|
||||
int ret;
|
||||
|
||||
ret = git_deflate_end_gently(stream);
|
||||
if (ret != Z_OK)
|
||||
return ret;
|
||||
git_hash_final_oid(oid, c);
|
||||
if (compat && compat_c)
|
||||
git_hash_final_oid(compat_oid, compat_c);
|
||||
|
||||
return Z_OK;
|
||||
}
|
||||
|
||||
int write_loose_object(struct odb_source_loose *loose,
|
||||
const struct object_id *oid, char *hdr,
|
||||
int hdrlen, const void *buf, unsigned long len,
|
||||
time_t mtime, unsigned flags)
|
||||
{
|
||||
int fd, ret;
|
||||
unsigned char compressed[4096];
|
||||
git_zstream stream;
|
||||
struct git_hash_ctx c;
|
||||
struct object_id parano_oid;
|
||||
static struct strbuf tmp_file = STRBUF_INIT;
|
||||
static struct strbuf filename = STRBUF_INIT;
|
||||
|
||||
if (batch_fsync_enabled(FSYNC_COMPONENT_LOOSE_OBJECT))
|
||||
odb_transaction_files_prepare(loose->base.odb->transaction);
|
||||
|
||||
odb_loose_path(loose, &filename, oid);
|
||||
|
||||
fd = start_loose_object_common(loose, &tmp_file, filename.buf, flags,
|
||||
&stream, compressed, sizeof(compressed),
|
||||
&c, NULL, hdr, hdrlen);
|
||||
if (fd < 0)
|
||||
return -1;
|
||||
|
||||
/* Then the data itself.. */
|
||||
stream.next_in = (void *)buf;
|
||||
stream.avail_in = len;
|
||||
do {
|
||||
unsigned char *in0 = stream.next_in;
|
||||
|
||||
ret = write_loose_object_common(loose, &c, NULL, &stream, 1, in0, fd,
|
||||
compressed, sizeof(compressed));
|
||||
} while (ret == Z_OK);
|
||||
|
||||
if (ret != Z_STREAM_END)
|
||||
die(_("unable to deflate new object %s (%d)"), oid_to_hex(oid),
|
||||
ret);
|
||||
ret = end_loose_object_common(loose, &c, NULL, &stream, ¶no_oid, NULL);
|
||||
if (ret != Z_OK)
|
||||
die(_("deflateEnd on object %s failed (%d)"), oid_to_hex(oid),
|
||||
ret);
|
||||
if (!oideq(oid, ¶no_oid))
|
||||
die(_("confused by unstable object source data for %s"),
|
||||
oid_to_hex(oid));
|
||||
|
||||
close_loose_object(loose, fd, tmp_file.buf);
|
||||
|
||||
if (mtime) {
|
||||
struct utimbuf utb;
|
||||
utb.actime = mtime;
|
||||
utb.modtime = mtime;
|
||||
if (utime(tmp_file.buf, &utb) < 0 &&
|
||||
!(flags & ODB_WRITE_OBJECT_SILENT))
|
||||
warning_errno(_("failed utime() on %s"), tmp_file.buf);
|
||||
}
|
||||
|
||||
return finalize_object_file_flags(loose->base.odb->repo, tmp_file.buf, filename.buf,
|
||||
FOF_SKIP_COLLISION_CHECK);
|
||||
}
|
||||
|
||||
int odb_source_loose_write_stream(struct odb_source_loose *loose,
|
||||
struct odb_write_stream *in_stream, size_t len,
|
||||
struct object_id *oid)
|
||||
{
|
||||
const struct git_hash_algo *compat = loose->base.odb->repo->compat_hash_algo;
|
||||
struct object_id compat_oid;
|
||||
int fd, ret, err = 0, flush = 0;
|
||||
unsigned char compressed[4096];
|
||||
git_zstream stream;
|
||||
struct git_hash_ctx c, compat_c;
|
||||
struct strbuf tmp_file = STRBUF_INIT;
|
||||
struct strbuf filename = STRBUF_INIT;
|
||||
unsigned char buf[8192];
|
||||
int dirlen;
|
||||
char hdr[MAX_HEADER_LEN];
|
||||
int hdrlen;
|
||||
|
||||
if (batch_fsync_enabled(FSYNC_COMPONENT_LOOSE_OBJECT))
|
||||
odb_transaction_files_prepare(loose->base.odb->transaction);
|
||||
|
||||
/* Since oid is not determined, save tmp file to odb path. */
|
||||
strbuf_addf(&filename, "%s/", loose->base.path);
|
||||
hdrlen = format_object_header(hdr, sizeof(hdr), OBJ_BLOB, len);
|
||||
|
||||
/*
|
||||
* Common steps for write_loose_object and stream_loose_object to
|
||||
* start writing loose objects:
|
||||
*
|
||||
* - Create tmpfile for the loose object.
|
||||
* - Setup zlib stream for compression.
|
||||
* - Start to feed header to zlib stream.
|
||||
*/
|
||||
fd = start_loose_object_common(loose, &tmp_file, filename.buf, 0,
|
||||
&stream, compressed, sizeof(compressed),
|
||||
&c, &compat_c, hdr, hdrlen);
|
||||
if (fd < 0) {
|
||||
err = -1;
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
/* Then the data itself.. */
|
||||
do {
|
||||
unsigned char *in0 = stream.next_in;
|
||||
|
||||
if (!stream.avail_in && !in_stream->is_finished) {
|
||||
ssize_t read_len = odb_write_stream_read(in_stream, buf,
|
||||
sizeof(buf));
|
||||
if (read_len < 0) {
|
||||
close(fd);
|
||||
err = -1;
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
stream.avail_in = read_len;
|
||||
stream.next_in = buf;
|
||||
in0 = buf;
|
||||
/* All data has been read. */
|
||||
if (in_stream->is_finished)
|
||||
flush = 1;
|
||||
}
|
||||
ret = write_loose_object_common(loose, &c, &compat_c, &stream, flush, in0, fd,
|
||||
compressed, sizeof(compressed));
|
||||
/*
|
||||
* Unlike write_loose_object(), we do not have the entire
|
||||
* buffer. If we get Z_BUF_ERROR due to too few input bytes,
|
||||
* then we'll replenish them in the next input_stream->read()
|
||||
* call when we loop.
|
||||
*/
|
||||
} while (ret == Z_OK || ret == Z_BUF_ERROR);
|
||||
|
||||
if (stream.total_in != len + hdrlen)
|
||||
die(_("write stream object %"PRIuMAX" != %"PRIuMAX), (uintmax_t)stream.total_in,
|
||||
(uintmax_t)len + hdrlen);
|
||||
|
||||
/*
|
||||
* Common steps for write_loose_object and stream_loose_object to
|
||||
* end writing loose object:
|
||||
*
|
||||
* - End the compression of zlib stream.
|
||||
* - Get the calculated oid.
|
||||
*/
|
||||
if (ret != Z_STREAM_END)
|
||||
die(_("unable to stream deflate new object (%d)"), ret);
|
||||
ret = end_loose_object_common(loose, &c, &compat_c, &stream, oid, &compat_oid);
|
||||
if (ret != Z_OK)
|
||||
die(_("deflateEnd on stream object failed (%d)"), ret);
|
||||
close_loose_object(loose, fd, tmp_file.buf);
|
||||
|
||||
if (odb_freshen_object(loose->base.odb, oid)) {
|
||||
unlink_or_warn(tmp_file.buf);
|
||||
goto cleanup;
|
||||
}
|
||||
odb_loose_path(loose, &filename, oid);
|
||||
|
||||
/* We finally know the object path, and create the missing dir. */
|
||||
dirlen = directory_size(filename.buf);
|
||||
if (dirlen) {
|
||||
struct strbuf dir = STRBUF_INIT;
|
||||
strbuf_add(&dir, filename.buf, dirlen);
|
||||
|
||||
if (safe_create_dir_in_gitdir(loose->base.odb->repo, dir.buf) &&
|
||||
errno != EEXIST) {
|
||||
err = error_errno(_("unable to create directory %s"), dir.buf);
|
||||
strbuf_release(&dir);
|
||||
goto cleanup;
|
||||
}
|
||||
strbuf_release(&dir);
|
||||
}
|
||||
|
||||
err = finalize_object_file_flags(loose->base.odb->repo, tmp_file.buf, filename.buf,
|
||||
FOF_SKIP_COLLISION_CHECK);
|
||||
if (!err && compat)
|
||||
err = repo_add_loose_object_map(loose, oid, &compat_oid);
|
||||
cleanup:
|
||||
strbuf_release(&tmp_file);
|
||||
strbuf_release(&filename);
|
||||
return err;
|
||||
}
|
||||
|
||||
int force_object_loose(struct odb_source *source,
|
||||
const struct object_id *oid, time_t mtime)
|
||||
{
|
||||
struct odb_source_files *files = odb_source_files_downcast(source);
|
||||
const struct git_hash_algo *compat = source->odb->repo->compat_hash_algo;
|
||||
void *buf;
|
||||
size_t len;
|
||||
struct object_info oi = OBJECT_INFO_INIT;
|
||||
struct object_id compat_oid;
|
||||
enum object_type type;
|
||||
char hdr[MAX_HEADER_LEN];
|
||||
int hdrlen;
|
||||
int ret;
|
||||
|
||||
for (struct odb_source *s = source->odb->sources; s; s = s->next) {
|
||||
struct odb_source_files *files = odb_source_files_downcast(s);
|
||||
if (!odb_source_read_object_info(&files->loose->base, oid, NULL, 0))
|
||||
return 0;
|
||||
}
|
||||
|
||||
oi.typep = &type;
|
||||
oi.sizep = &len;
|
||||
oi.contentp = &buf;
|
||||
if (odb_read_object_info_extended(source->odb, oid, &oi, 0))
|
||||
return error(_("cannot read object for %s"), oid_to_hex(oid));
|
||||
if (compat) {
|
||||
if (repo_oid_to_algop(source->odb->repo, oid, compat, &compat_oid))
|
||||
return error(_("cannot map object %s to %s"),
|
||||
oid_to_hex(oid), compat->name);
|
||||
}
|
||||
hdrlen = format_object_header(hdr, sizeof(hdr), type, len);
|
||||
ret = write_loose_object(files->loose, oid, hdr, hdrlen, buf, len, mtime, 0);
|
||||
if (!ret && compat)
|
||||
ret = repo_add_loose_object_map(files->loose, oid, &compat_oid);
|
||||
free(buf);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
/*
|
||||
* We can't use the normal fsck_error_function() for index_mem(),
|
||||
* because we don't yet have a valid oid for it to report. Instead,
|
||||
|
|
|
|||
|
|
@ -24,20 +24,6 @@ int index_path(struct index_state *istate, struct object_id *oid, const char *pa
|
|||
struct object_info;
|
||||
struct odb_source;
|
||||
|
||||
/*
|
||||
* Write the given stream into the loose object source. The only difference
|
||||
* from the generic implementation of this function is that we don't perform an
|
||||
* object existence check here.
|
||||
*
|
||||
* TODO: We should stop exposing this function altogether and move it into
|
||||
* "odb/source-loose.c". This requires a couple of refactorings though to make
|
||||
* `force_object_loose()` generic and is thus postponed to a later point in
|
||||
* time.
|
||||
*/
|
||||
int odb_source_loose_write_stream(struct odb_source_loose *source,
|
||||
struct odb_write_stream *stream, size_t len,
|
||||
struct object_id *oid);
|
||||
|
||||
/*
|
||||
* Put in `buf` the name of the file in the local object database that
|
||||
* would be used to store a loose object with the specified oid.
|
||||
|
|
@ -98,9 +84,6 @@ int for_each_file_in_obj_subdir(unsigned int subdir_nr,
|
|||
int format_object_header(char *str, size_t size, enum object_type type,
|
||||
size_t objsize);
|
||||
|
||||
int force_object_loose(struct odb_source *source,
|
||||
const struct object_id *oid, time_t mtime);
|
||||
|
||||
/**
|
||||
* With in-core object data in "buf", rehash it to make sure the
|
||||
* object name actually matches "oid" to detect object corruption.
|
||||
|
|
@ -134,17 +117,10 @@ int finalize_object_file_flags(struct repository *repo,
|
|||
void hash_object_file(const struct git_hash_algo *algo, const void *buf,
|
||||
size_t len, enum object_type type,
|
||||
struct object_id *oid);
|
||||
void write_object_file_prepare(const struct git_hash_algo *algo,
|
||||
const void *buf, size_t len,
|
||||
enum object_type type, struct object_id *oid,
|
||||
char *hdr, size_t *hdrlen);
|
||||
int write_loose_object(struct odb_source_loose *loose,
|
||||
const struct object_id *oid, char *hdr,
|
||||
int hdrlen, const void *buf, unsigned long len,
|
||||
time_t mtime, unsigned flags);
|
||||
|
||||
/* Helper to check and "touch" a file */
|
||||
int check_and_freshen_file(const char *fn, int freshen);
|
||||
int check_and_freshen_file(const char *fn, int freshen,
|
||||
const time_t *mtime);
|
||||
|
||||
/*
|
||||
* Open the loose object at path, check its hash, and return the contents,
|
||||
|
|
@ -201,4 +177,8 @@ int odb_transaction_files_begin(struct odb_source *source,
|
|||
struct odb_transaction **out,
|
||||
enum odb_transaction_flags flags);
|
||||
|
||||
int odb_transaction_files_prepare(struct odb_transaction *base);
|
||||
void odb_transaction_files_fsync(struct odb_transaction *base,
|
||||
int fd, const char *filename);
|
||||
|
||||
#endif /* OBJECT_FILE_H */
|
||||
|
|
|
|||
39
odb.c
39
odb.c
|
|
@ -738,7 +738,7 @@ int odb_pretend_object(struct object_database *odb,
|
|||
return 0;
|
||||
|
||||
return odb_source_write_object(odb->inmemory_objects,
|
||||
buf, len, type, oid, NULL, 0);
|
||||
buf, len, type, oid, NULL, NULL, 0);
|
||||
}
|
||||
|
||||
void *odb_read_object(struct object_database *odb,
|
||||
|
|
@ -829,7 +829,7 @@ int odb_freshen_object(struct object_database *odb,
|
|||
struct odb_source *source;
|
||||
odb_prepare_alternates(odb);
|
||||
for (source = odb->sources; source; source = source->next)
|
||||
if (odb_source_freshen_object(source, oid))
|
||||
if (odb_source_freshen_object(source, oid, NULL))
|
||||
return 1;
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -989,11 +989,42 @@ int odb_write_object_ext(struct object_database *odb,
|
|||
const void *buf, unsigned long len,
|
||||
enum object_type type,
|
||||
struct object_id *oid,
|
||||
struct object_id *compat_oid,
|
||||
const struct object_id *compat_oid_in,
|
||||
enum odb_write_object_flags flags)
|
||||
{
|
||||
const struct git_hash_algo *compat = odb->repo->compat_hash_algo;
|
||||
struct object_id compat_oid, *compat_oid_p = NULL;
|
||||
|
||||
hash_object_file(odb->repo->hash_algo, buf, len, type, oid);
|
||||
|
||||
/*
|
||||
* We can skip the write in case we already have the object available.
|
||||
* In that case, we only freshen its mtime.
|
||||
*/
|
||||
if (odb_freshen_object(odb, oid))
|
||||
return 0;
|
||||
|
||||
if (compat) {
|
||||
const struct git_hash_algo *algo = odb->repo->hash_algo;
|
||||
|
||||
if (compat_oid_in) {
|
||||
oidcpy(&compat_oid, compat_oid_in);
|
||||
} else if (type == OBJ_BLOB) {
|
||||
hash_object_file(compat, buf, len, type, &compat_oid);
|
||||
} else {
|
||||
struct strbuf converted = STRBUF_INIT;
|
||||
convert_object_file(odb->repo, &converted, algo, compat,
|
||||
buf, len, type, 0);
|
||||
hash_object_file(compat, converted.buf, converted.len,
|
||||
type, &compat_oid);
|
||||
strbuf_release(&converted);
|
||||
}
|
||||
|
||||
compat_oid_p = &compat_oid;
|
||||
}
|
||||
|
||||
return odb_source_write_object(odb->sources, buf, len, type,
|
||||
oid, compat_oid, flags);
|
||||
oid, compat_oid_p, NULL, flags);
|
||||
}
|
||||
|
||||
int odb_write_object_stream(struct object_database *odb,
|
||||
|
|
|
|||
10
odb.h
10
odb.h
|
|
@ -603,9 +603,11 @@ enum odb_write_object_flags {
|
|||
|
||||
/*
|
||||
* Write an object into the object database. The object is being written into
|
||||
* the local alternate of the repository. If provided, the converted object ID
|
||||
* as well as the compatibility object ID are written to the respective
|
||||
* pointers.
|
||||
* the local alternate of the repository. If provided, the object ID of the
|
||||
* final object is written into `oid`.
|
||||
*
|
||||
* If the caller provides a `compat_oid`, then this compatibility object hash
|
||||
* will be stored instead of computing the compatibility hash ad-hoc.
|
||||
*
|
||||
* Returns 0 on success, a negative error code otherwise.
|
||||
*/
|
||||
|
|
@ -613,7 +615,7 @@ int odb_write_object_ext(struct object_database *odb,
|
|||
const void *buf, unsigned long len,
|
||||
enum object_type type,
|
||||
struct object_id *oid,
|
||||
struct object_id *compat_oid,
|
||||
const struct object_id *compat_oid,
|
||||
enum odb_write_object_flags flags);
|
||||
|
||||
static inline int odb_write_object(struct object_database *odb,
|
||||
|
|
|
|||
|
|
@ -150,11 +150,12 @@ out:
|
|||
}
|
||||
|
||||
static int odb_source_files_freshen_object(struct odb_source *source,
|
||||
const struct object_id *oid)
|
||||
const struct object_id *oid,
|
||||
const time_t *mtime)
|
||||
{
|
||||
struct odb_source_files *files = odb_source_files_downcast(source);
|
||||
if (odb_source_freshen_object(&files->packed->base, oid) ||
|
||||
odb_source_freshen_object(&files->loose->base, oid))
|
||||
if (odb_source_freshen_object(&files->packed->base, oid, mtime) ||
|
||||
odb_source_freshen_object(&files->loose->base, oid, mtime))
|
||||
return 1;
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -162,13 +163,14 @@ static int odb_source_files_freshen_object(struct odb_source *source,
|
|||
static int odb_source_files_write_object(struct odb_source *source,
|
||||
const void *buf, size_t len,
|
||||
enum object_type type,
|
||||
struct object_id *oid,
|
||||
struct object_id *compat_oid,
|
||||
const struct object_id *oid,
|
||||
const struct object_id *compat_oid,
|
||||
const time_t *mtime,
|
||||
enum odb_write_object_flags flags)
|
||||
{
|
||||
struct odb_source_files *files = odb_source_files_downcast(source);
|
||||
return odb_source_write_object(&files->loose->base, buf, len, type,
|
||||
oid, compat_oid, flags);
|
||||
oid, compat_oid, mtime, flags);
|
||||
}
|
||||
|
||||
static int odb_source_files_write_object_stream(struct odb_source *source,
|
||||
|
|
|
|||
|
|
@ -230,15 +230,14 @@ static int odb_source_inmemory_count_objects(struct odb_source *source,
|
|||
static int odb_source_inmemory_write_object(struct odb_source *source,
|
||||
const void *buf, size_t len,
|
||||
enum object_type type,
|
||||
struct object_id *oid,
|
||||
struct object_id *compat_oid UNUSED,
|
||||
const struct object_id *oid,
|
||||
const struct object_id *compat_oid UNUSED,
|
||||
const time_t *mtime UNUSED,
|
||||
enum odb_write_object_flags flags UNUSED)
|
||||
{
|
||||
struct odb_source_inmemory *inmemory = odb_source_inmemory_downcast(source);
|
||||
struct inmemory_object *object;
|
||||
|
||||
hash_object_file(source->odb->repo->hash_algo, buf, len, type, oid);
|
||||
|
||||
if (!inmemory->objects) {
|
||||
CALLOC_ARRAY(inmemory->objects, 1);
|
||||
oidtree_init(inmemory->objects);
|
||||
|
|
@ -285,8 +284,10 @@ static int odb_source_inmemory_write_object_stream(struct odb_source *source,
|
|||
goto out;
|
||||
}
|
||||
|
||||
hash_object_file(source->odb->repo->hash_algo, data, total_read, OBJ_BLOB, oid);
|
||||
|
||||
ret = odb_source_inmemory_write_object(source, data, len, OBJ_BLOB, oid,
|
||||
NULL, 0);
|
||||
NULL, NULL, 0);
|
||||
if (ret < 0)
|
||||
goto out;
|
||||
|
||||
|
|
@ -296,7 +297,8 @@ out:
|
|||
}
|
||||
|
||||
static int odb_source_inmemory_freshen_object(struct odb_source *source,
|
||||
const struct object_id *oid)
|
||||
const struct object_id *oid,
|
||||
const time_t *mtime UNUSED)
|
||||
{
|
||||
struct odb_source_inmemory *inmemory = odb_source_inmemory_downcast(source);
|
||||
if (find_cached_object(inmemory, oid))
|
||||
|
|
|
|||
|
|
@ -11,8 +11,11 @@
|
|||
#include "odb/source-loose.h"
|
||||
#include "odb/streaming.h"
|
||||
#include "oidtree.h"
|
||||
#include "path.h"
|
||||
#include "repository.h"
|
||||
#include "strbuf.h"
|
||||
#include "tempfile.h"
|
||||
#include "write-or-die.h"
|
||||
|
||||
static int append_loose_object(const struct object_id *oid,
|
||||
const char *path UNUSED,
|
||||
|
|
@ -574,53 +577,270 @@ out:
|
|||
}
|
||||
|
||||
static int odb_source_loose_freshen_object(struct odb_source *source,
|
||||
const struct object_id *oid)
|
||||
const struct object_id *oid,
|
||||
const time_t *mtime)
|
||||
{
|
||||
struct odb_source_loose *loose = odb_source_loose_downcast(source);
|
||||
static struct strbuf path = STRBUF_INIT;
|
||||
odb_loose_path(loose, &path, oid);
|
||||
return !!check_and_freshen_file(path.buf, 1);
|
||||
return !!check_and_freshen_file(path.buf, 1, mtime);
|
||||
}
|
||||
|
||||
/* Finalize a file on disk, and close it. */
|
||||
static void close_loose_object(struct odb_source_loose *loose,
|
||||
int fd, const char *filename)
|
||||
{
|
||||
if (loose->base.will_destroy)
|
||||
goto out;
|
||||
|
||||
if (batch_fsync_enabled(FSYNC_COMPONENT_LOOSE_OBJECT))
|
||||
odb_transaction_files_fsync(loose->base.odb->transaction, fd, filename);
|
||||
else if (fsync_object_files > 0)
|
||||
fsync_or_die(fd, filename);
|
||||
else
|
||||
fsync_component_or_die(FSYNC_COMPONENT_LOOSE_OBJECT, fd,
|
||||
filename);
|
||||
|
||||
out:
|
||||
if (close(fd) != 0)
|
||||
die_errno(_("error when closing loose object file"));
|
||||
}
|
||||
|
||||
/* Size of directory component, including the ending '/' */
|
||||
static inline int directory_size(const char *filename)
|
||||
{
|
||||
const char *s = strrchr(filename, '/');
|
||||
if (!s)
|
||||
return 0;
|
||||
return s - filename + 1;
|
||||
}
|
||||
|
||||
/*
|
||||
* This creates a temporary file in the same directory as the final
|
||||
* 'filename'
|
||||
*
|
||||
* We want to avoid cross-directory filename renames, because those
|
||||
* can have problems on various filesystems (FAT, NFS, Coda).
|
||||
*/
|
||||
static int create_tmpfile(struct repository *repo,
|
||||
struct strbuf *tmp, const char *filename)
|
||||
{
|
||||
int fd, dirlen = directory_size(filename);
|
||||
|
||||
strbuf_reset(tmp);
|
||||
strbuf_add(tmp, filename, dirlen);
|
||||
strbuf_addstr(tmp, "tmp_obj_XXXXXX");
|
||||
fd = git_mkstemp_mode(tmp->buf, 0444);
|
||||
if (fd < 0 && dirlen && errno == ENOENT) {
|
||||
/*
|
||||
* Make sure the directory exists; note that the contents
|
||||
* of the buffer are undefined after mkstemp returns an
|
||||
* error, so we have to rewrite the whole buffer from
|
||||
* scratch.
|
||||
*/
|
||||
strbuf_reset(tmp);
|
||||
strbuf_add(tmp, filename, dirlen - 1);
|
||||
if (mkdir(tmp->buf, 0777) && errno != EEXIST)
|
||||
return -1;
|
||||
if (adjust_shared_perm(repo, tmp->buf))
|
||||
return -1;
|
||||
|
||||
/* Try again */
|
||||
strbuf_addstr(tmp, "/tmp_obj_XXXXXX");
|
||||
fd = git_mkstemp_mode(tmp->buf, 0444);
|
||||
}
|
||||
return fd;
|
||||
}
|
||||
|
||||
/**
|
||||
* Common steps for loose object writers to start writing loose
|
||||
* objects:
|
||||
*
|
||||
* - Create tmpfile for the loose object.
|
||||
* - Setup zlib stream for compression.
|
||||
* - Start to feed header to zlib stream.
|
||||
*
|
||||
* Returns a "fd", which should later be provided to
|
||||
* end_loose_object_common().
|
||||
*/
|
||||
static int start_loose_object_common(struct odb_source_loose *loose,
|
||||
struct strbuf *tmp_file,
|
||||
const char *filename, unsigned flags,
|
||||
git_zstream *stream,
|
||||
unsigned char *buf, size_t buflen,
|
||||
struct git_hash_ctx *c, struct git_hash_ctx *compat_c,
|
||||
char *hdr, int hdrlen)
|
||||
{
|
||||
const struct git_hash_algo *algo = loose->base.odb->repo->hash_algo;
|
||||
const struct git_hash_algo *compat = loose->base.odb->repo->compat_hash_algo;
|
||||
int fd;
|
||||
struct repo_config_values *cfg = repo_config_values(loose->base.odb->repo);
|
||||
|
||||
fd = create_tmpfile(loose->base.odb->repo, tmp_file, filename);
|
||||
if (fd < 0) {
|
||||
if (flags & ODB_WRITE_OBJECT_SILENT)
|
||||
return -1;
|
||||
else if (errno == EACCES)
|
||||
return error(_("insufficient permission for adding "
|
||||
"an object to repository database %s"),
|
||||
loose->base.path);
|
||||
else
|
||||
return error_errno(
|
||||
_("unable to create temporary file"));
|
||||
}
|
||||
|
||||
/* Setup zlib stream for compression */
|
||||
git_deflate_init(stream, cfg->zlib_compression_level);
|
||||
stream->next_out = buf;
|
||||
stream->avail_out = buflen;
|
||||
git_hash_init(c, algo);
|
||||
if (compat && compat_c)
|
||||
git_hash_init(compat_c, compat);
|
||||
|
||||
/* Start to feed header to zlib stream */
|
||||
stream->next_in = (unsigned char *)hdr;
|
||||
stream->avail_in = hdrlen;
|
||||
while (git_deflate(stream, 0) == Z_OK)
|
||||
; /* nothing */
|
||||
git_hash_update(c, hdr, hdrlen);
|
||||
if (compat && compat_c)
|
||||
git_hash_update(compat_c, hdr, hdrlen);
|
||||
|
||||
return fd;
|
||||
}
|
||||
|
||||
/**
|
||||
* Common steps for the inner git_deflate() loop for writing loose
|
||||
* objects. Returns what git_deflate() returns.
|
||||
*/
|
||||
static int write_loose_object_common(struct odb_source_loose *loose,
|
||||
struct git_hash_ctx *c, struct git_hash_ctx *compat_c,
|
||||
git_zstream *stream, const int flush,
|
||||
unsigned char *in0, const int fd,
|
||||
unsigned char *compressed,
|
||||
const size_t compressed_len)
|
||||
{
|
||||
const struct git_hash_algo *compat = loose->base.odb->repo->compat_hash_algo;
|
||||
int ret;
|
||||
|
||||
ret = git_deflate(stream, flush ? Z_FINISH : 0);
|
||||
git_hash_update(c, in0, stream->next_in - in0);
|
||||
if (compat && compat_c)
|
||||
git_hash_update(compat_c, in0, stream->next_in - in0);
|
||||
if (write_in_full(fd, compressed, stream->next_out - compressed) < 0)
|
||||
die_errno(_("unable to write loose object file"));
|
||||
stream->next_out = compressed;
|
||||
stream->avail_out = compressed_len;
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
/**
|
||||
* Common steps for loose object writers to end writing loose objects:
|
||||
*
|
||||
* - End the compression of zlib stream.
|
||||
* - Get the calculated oid to "oid".
|
||||
*/
|
||||
static int end_loose_object_common(struct odb_source_loose *loose,
|
||||
struct git_hash_ctx *c, struct git_hash_ctx *compat_c,
|
||||
git_zstream *stream, struct object_id *oid,
|
||||
struct object_id *compat_oid)
|
||||
{
|
||||
const struct git_hash_algo *compat = loose->base.odb->repo->compat_hash_algo;
|
||||
int ret;
|
||||
|
||||
ret = git_deflate_end_gently(stream);
|
||||
if (ret != Z_OK)
|
||||
return ret;
|
||||
git_hash_final_oid(oid, c);
|
||||
if (compat && compat_c)
|
||||
git_hash_final_oid(compat_oid, compat_c);
|
||||
|
||||
return Z_OK;
|
||||
}
|
||||
|
||||
static int write_loose_object(struct odb_source_loose *loose,
|
||||
const struct object_id *oid, char *hdr,
|
||||
int hdrlen, const void *buf, unsigned long len,
|
||||
const time_t *mtime, unsigned flags)
|
||||
{
|
||||
int fd, ret;
|
||||
unsigned char compressed[4096];
|
||||
git_zstream stream;
|
||||
struct git_hash_ctx c;
|
||||
struct object_id parano_oid;
|
||||
static struct strbuf tmp_file = STRBUF_INIT;
|
||||
static struct strbuf filename = STRBUF_INIT;
|
||||
|
||||
if (batch_fsync_enabled(FSYNC_COMPONENT_LOOSE_OBJECT))
|
||||
odb_transaction_files_prepare(loose->base.odb->transaction);
|
||||
|
||||
odb_loose_path(loose, &filename, oid);
|
||||
|
||||
fd = start_loose_object_common(loose, &tmp_file, filename.buf, flags,
|
||||
&stream, compressed, sizeof(compressed),
|
||||
&c, NULL, hdr, hdrlen);
|
||||
if (fd < 0)
|
||||
return -1;
|
||||
|
||||
/* Then the data itself.. */
|
||||
stream.next_in = (void *)buf;
|
||||
stream.avail_in = len;
|
||||
do {
|
||||
unsigned char *in0 = stream.next_in;
|
||||
|
||||
ret = write_loose_object_common(loose, &c, NULL, &stream, 1, in0, fd,
|
||||
compressed, sizeof(compressed));
|
||||
} while (ret == Z_OK);
|
||||
|
||||
if (ret != Z_STREAM_END)
|
||||
die(_("unable to deflate new object %s (%d)"), oid_to_hex(oid),
|
||||
ret);
|
||||
ret = end_loose_object_common(loose, &c, NULL, &stream, ¶no_oid, NULL);
|
||||
if (ret != Z_OK)
|
||||
die(_("deflateEnd on object %s failed (%d)"), oid_to_hex(oid),
|
||||
ret);
|
||||
if (!oideq(oid, ¶no_oid))
|
||||
die(_("confused by unstable object source data for %s"),
|
||||
oid_to_hex(oid));
|
||||
|
||||
close_loose_object(loose, fd, tmp_file.buf);
|
||||
|
||||
if (mtime) {
|
||||
struct utimbuf utb = {
|
||||
.actime = *mtime,
|
||||
.modtime = *mtime,
|
||||
};
|
||||
|
||||
if (utime(tmp_file.buf, &utb) < 0 &&
|
||||
!(flags & ODB_WRITE_OBJECT_SILENT))
|
||||
warning_errno(_("failed utime() on %s"), tmp_file.buf);
|
||||
}
|
||||
|
||||
return finalize_object_file_flags(loose->base.odb->repo, tmp_file.buf, filename.buf,
|
||||
FOF_SKIP_COLLISION_CHECK);
|
||||
}
|
||||
|
||||
static int odb_source_loose_write_object(struct odb_source *source,
|
||||
const void *buf, size_t len,
|
||||
enum object_type type, struct object_id *oid,
|
||||
struct object_id *compat_oid_in,
|
||||
enum object_type type,
|
||||
const struct object_id *oid,
|
||||
const struct object_id *compat_oid,
|
||||
const time_t *mtime,
|
||||
enum odb_write_object_flags flags)
|
||||
{
|
||||
struct odb_source_loose *loose = odb_source_loose_downcast(source);
|
||||
const struct git_hash_algo *algo = source->odb->repo->hash_algo;
|
||||
const struct git_hash_algo *compat = source->odb->repo->compat_hash_algo;
|
||||
struct object_id compat_oid;
|
||||
char hdr[MAX_HEADER_LEN];
|
||||
size_t hdrlen = sizeof(hdr);
|
||||
int hdrlen;
|
||||
|
||||
/* Generate compat_oid */
|
||||
if (compat) {
|
||||
if (compat_oid_in)
|
||||
oidcpy(&compat_oid, compat_oid_in);
|
||||
else if (type == OBJ_BLOB)
|
||||
hash_object_file(compat, buf, len, type, &compat_oid);
|
||||
else {
|
||||
struct strbuf converted = STRBUF_INIT;
|
||||
convert_object_file(source->odb->repo, &converted, algo, compat,
|
||||
buf, len, type, 0);
|
||||
hash_object_file(compat, converted.buf, converted.len,
|
||||
type, &compat_oid);
|
||||
strbuf_release(&converted);
|
||||
}
|
||||
}
|
||||
hdrlen = format_object_header(hdr, sizeof(hdr), type, len);
|
||||
|
||||
/* Normally if we have it in the pack then we do not bother writing
|
||||
* it out into .git/objects/??/?{38} file.
|
||||
*/
|
||||
write_object_file_prepare(algo, buf, len, type, oid, hdr, &hdrlen);
|
||||
if (odb_freshen_object(source->odb, oid))
|
||||
return 0;
|
||||
if (write_loose_object(loose, oid, hdr, hdrlen, buf, len, 0, flags))
|
||||
if (write_loose_object(loose, oid, hdr, hdrlen, buf, len, mtime, flags))
|
||||
return -1;
|
||||
if (compat)
|
||||
return repo_add_loose_object_map(loose, oid, &compat_oid);
|
||||
|
||||
if (compat_oid)
|
||||
return repo_add_loose_object_map(loose, oid, compat_oid);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -629,12 +849,120 @@ static int odb_source_loose_write_object_stream(struct odb_source *source,
|
|||
size_t len,
|
||||
struct object_id *oid)
|
||||
{
|
||||
/*
|
||||
* TODO: the implementation should be moved here, see the comment on
|
||||
* the called function in "object-file.h".
|
||||
*/
|
||||
struct odb_source_loose *loose = odb_source_loose_downcast(source);
|
||||
return odb_source_loose_write_stream(loose, in_stream, len, oid);
|
||||
const struct git_hash_algo *compat = loose->base.odb->repo->compat_hash_algo;
|
||||
struct object_id compat_oid;
|
||||
int fd, ret, err = 0, flush = 0;
|
||||
unsigned char compressed[4096];
|
||||
git_zstream stream;
|
||||
struct git_hash_ctx c, compat_c;
|
||||
struct strbuf tmp_file = STRBUF_INIT;
|
||||
struct strbuf filename = STRBUF_INIT;
|
||||
unsigned char buf[8192];
|
||||
int dirlen;
|
||||
char hdr[MAX_HEADER_LEN];
|
||||
int hdrlen;
|
||||
|
||||
if (batch_fsync_enabled(FSYNC_COMPONENT_LOOSE_OBJECT))
|
||||
odb_transaction_files_prepare(loose->base.odb->transaction);
|
||||
|
||||
/* Since oid is not determined, save tmp file to odb path. */
|
||||
strbuf_addf(&filename, "%s/", loose->base.path);
|
||||
hdrlen = format_object_header(hdr, sizeof(hdr), OBJ_BLOB, len);
|
||||
|
||||
/*
|
||||
* Common steps for write_loose_object and stream_loose_object to
|
||||
* start writing loose objects:
|
||||
*
|
||||
* - Create tmpfile for the loose object.
|
||||
* - Setup zlib stream for compression.
|
||||
* - Start to feed header to zlib stream.
|
||||
*/
|
||||
fd = start_loose_object_common(loose, &tmp_file, filename.buf, 0,
|
||||
&stream, compressed, sizeof(compressed),
|
||||
&c, &compat_c, hdr, hdrlen);
|
||||
if (fd < 0) {
|
||||
err = -1;
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
/* Then the data itself.. */
|
||||
do {
|
||||
unsigned char *in0 = stream.next_in;
|
||||
|
||||
if (!stream.avail_in && !in_stream->is_finished) {
|
||||
ssize_t read_len = odb_write_stream_read(in_stream, buf,
|
||||
sizeof(buf));
|
||||
if (read_len < 0) {
|
||||
close(fd);
|
||||
err = -1;
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
stream.avail_in = read_len;
|
||||
stream.next_in = buf;
|
||||
in0 = buf;
|
||||
/* All data has been read. */
|
||||
if (in_stream->is_finished)
|
||||
flush = 1;
|
||||
}
|
||||
ret = write_loose_object_common(loose, &c, &compat_c, &stream, flush, in0, fd,
|
||||
compressed, sizeof(compressed));
|
||||
/*
|
||||
* Unlike write_loose_object(), we do not have the entire
|
||||
* buffer. If we get Z_BUF_ERROR due to too few input bytes,
|
||||
* then we'll replenish them in the next input_stream->read()
|
||||
* call when we loop.
|
||||
*/
|
||||
} while (ret == Z_OK || ret == Z_BUF_ERROR);
|
||||
|
||||
if (stream.total_in != len + hdrlen)
|
||||
die(_("write stream object %"PRIuMAX" != %"PRIuMAX), (uintmax_t)stream.total_in,
|
||||
(uintmax_t)len + hdrlen);
|
||||
|
||||
/*
|
||||
* Common steps for write_loose_object and stream_loose_object to
|
||||
* end writing loose object:
|
||||
*
|
||||
* - End the compression of zlib stream.
|
||||
* - Get the calculated oid.
|
||||
*/
|
||||
if (ret != Z_STREAM_END)
|
||||
die(_("unable to stream deflate new object (%d)"), ret);
|
||||
ret = end_loose_object_common(loose, &c, &compat_c, &stream, oid, &compat_oid);
|
||||
if (ret != Z_OK)
|
||||
die(_("deflateEnd on stream object failed (%d)"), ret);
|
||||
close_loose_object(loose, fd, tmp_file.buf);
|
||||
|
||||
if (odb_freshen_object(loose->base.odb, oid)) {
|
||||
unlink_or_warn(tmp_file.buf);
|
||||
goto cleanup;
|
||||
}
|
||||
odb_loose_path(loose, &filename, oid);
|
||||
|
||||
/* We finally know the object path, and create the missing dir. */
|
||||
dirlen = directory_size(filename.buf);
|
||||
if (dirlen) {
|
||||
struct strbuf dir = STRBUF_INIT;
|
||||
strbuf_add(&dir, filename.buf, dirlen);
|
||||
|
||||
if (safe_create_dir_in_gitdir(loose->base.odb->repo, dir.buf) &&
|
||||
errno != EEXIST) {
|
||||
err = error_errno(_("unable to create directory %s"), dir.buf);
|
||||
strbuf_release(&dir);
|
||||
goto cleanup;
|
||||
}
|
||||
strbuf_release(&dir);
|
||||
}
|
||||
|
||||
err = finalize_object_file_flags(loose->base.odb->repo, tmp_file.buf, filename.buf,
|
||||
FOF_SKIP_COLLISION_CHECK);
|
||||
if (!err && compat)
|
||||
err = repo_add_loose_object_map(loose, oid, &compat_oid);
|
||||
cleanup:
|
||||
strbuf_release(&tmp_file);
|
||||
strbuf_release(&filename);
|
||||
return err;
|
||||
}
|
||||
|
||||
static int odb_source_loose_begin_transaction(struct odb_source *source UNUSED,
|
||||
|
|
|
|||
|
|
@ -570,18 +570,26 @@ static int odb_source_packed_find_abbrev_len(struct odb_source *source,
|
|||
}
|
||||
|
||||
static int odb_source_packed_freshen_object(struct odb_source *source,
|
||||
const struct object_id *oid)
|
||||
const struct object_id *oid,
|
||||
const time_t *mtime)
|
||||
{
|
||||
struct odb_source_packed *packed = odb_source_packed_downcast(source);
|
||||
struct utimbuf times, *timesp = NULL;
|
||||
struct pack_entry e;
|
||||
|
||||
if (mtime) {
|
||||
times.actime = *mtime;
|
||||
times.modtime = *mtime;
|
||||
timesp = ×
|
||||
}
|
||||
|
||||
if (!find_pack_entry(packed, oid, &e))
|
||||
return 0;
|
||||
if (e.p->is_cruft)
|
||||
return 0;
|
||||
if (e.p->freshened)
|
||||
return 1;
|
||||
if (utime(e.p->pack_name, NULL))
|
||||
if (utime(e.p->pack_name, timesp))
|
||||
return 0;
|
||||
e.p->freshened = 1;
|
||||
|
||||
|
|
@ -592,8 +600,9 @@ static int odb_source_packed_write_object(struct odb_source *source UNUSED,
|
|||
const void *buf UNUSED,
|
||||
size_t len UNUSED,
|
||||
enum object_type type UNUSED,
|
||||
struct object_id *oid UNUSED,
|
||||
struct object_id *compat_oid UNUSED,
|
||||
const struct object_id *oid UNUSED,
|
||||
const struct object_id *compat_oid UNUSED,
|
||||
const time_t *mtime UNUSED,
|
||||
unsigned flags UNUSED)
|
||||
{
|
||||
return error("packed backend cannot write objects");
|
||||
|
|
|
|||
20
odb/source.h
20
odb/source.h
|
|
@ -190,7 +190,8 @@ struct odb_source {
|
|||
* has been freshened.
|
||||
*/
|
||||
int (*freshen_object)(struct odb_source *source,
|
||||
const struct object_id *oid);
|
||||
const struct object_id *oid,
|
||||
const time_t *mtime);
|
||||
|
||||
/*
|
||||
* This callback is expected to persist the given object into the
|
||||
|
|
@ -206,8 +207,9 @@ struct odb_source {
|
|||
int (*write_object)(struct odb_source *source,
|
||||
const void *buf, size_t len,
|
||||
enum object_type type,
|
||||
struct object_id *oid,
|
||||
struct object_id *compat_oid,
|
||||
const struct object_id *oid,
|
||||
const struct object_id *compat_oid,
|
||||
const time_t *mtime,
|
||||
enum odb_write_object_flags flags);
|
||||
|
||||
/*
|
||||
|
|
@ -403,9 +405,10 @@ static inline int odb_source_find_abbrev_len(struct odb_source *source,
|
|||
* not exist.
|
||||
*/
|
||||
static inline int odb_source_freshen_object(struct odb_source *source,
|
||||
const struct object_id *oid)
|
||||
const struct object_id *oid,
|
||||
const time_t *mtime)
|
||||
{
|
||||
return source->freshen_object(source, oid);
|
||||
return source->freshen_object(source, oid, mtime);
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
@ -416,12 +419,13 @@ static inline int odb_source_freshen_object(struct odb_source *source,
|
|||
static inline int odb_source_write_object(struct odb_source *source,
|
||||
const void *buf, unsigned long len,
|
||||
enum object_type type,
|
||||
struct object_id *oid,
|
||||
struct object_id *compat_oid,
|
||||
const struct object_id *oid,
|
||||
const struct object_id *compat_oid,
|
||||
const time_t *mtime,
|
||||
enum odb_write_object_flags flags)
|
||||
{
|
||||
return source->write_object(source, buf, len, type, oid,
|
||||
compat_oid, flags);
|
||||
compat_oid, mtime, flags);
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
|
|||
|
|
@ -2341,7 +2341,7 @@ unmap:
|
|||
*/
|
||||
static void freshen_shared_index(const char *shared_index, int warn)
|
||||
{
|
||||
if (!check_and_freshen_file(shared_index, 1) && warn)
|
||||
if (!check_and_freshen_file(shared_index, 1, NULL) && warn)
|
||||
warning(_("could not freshen shared index '%s'"), shared_index);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
#include "unit-test.h"
|
||||
#include "hex.h"
|
||||
#include "object-file.h"
|
||||
#include "odb/source-inmemory.h"
|
||||
#include "odb/streaming.h"
|
||||
#include "oidset.h"
|
||||
|
|
@ -36,6 +37,17 @@ static void cl_assert_object_info(struct odb_source_inmemory *source,
|
|||
free(actual_content);
|
||||
}
|
||||
|
||||
static void cl_assert_write_object(struct odb_source_inmemory *source,
|
||||
const char *content,
|
||||
enum object_type type,
|
||||
struct object_id *oid)
|
||||
{
|
||||
size_t content_len = strlen(content);
|
||||
hash_object_file(repo.hash_algo, content, content_len, type, oid);
|
||||
cl_must_pass(odb_source_write_object(&source->base, content, content_len,
|
||||
type, oid, NULL, NULL, 0));
|
||||
}
|
||||
|
||||
void test_odb_inmemory__initialize(void)
|
||||
{
|
||||
odb = odb_new(&repo, "", "");
|
||||
|
|
@ -78,8 +90,7 @@ void test_odb_inmemory__read_written_object(void)
|
|||
const char data[] = "foobar";
|
||||
struct object_id written_oid;
|
||||
|
||||
cl_must_pass(odb_source_write_object(&source->base, data, strlen(data),
|
||||
OBJ_BLOB, &written_oid, NULL, 0));
|
||||
cl_assert_write_object(source, data, OBJ_BLOB, &written_oid);
|
||||
cl_assert_equal_s(oid_to_hex(&written_oid), FOOBAR_OID);
|
||||
cl_assert_object_info(source, &written_oid, OBJ_BLOB, "foobar");
|
||||
|
||||
|
|
@ -94,8 +105,7 @@ void test_odb_inmemory__read_stream_object(void)
|
|||
const char data[] = "foobar";
|
||||
char buf[3] = { 0 };
|
||||
|
||||
cl_must_pass(odb_source_write_object(&source->base, data, strlen(data),
|
||||
OBJ_BLOB, &written_oid, NULL, 0));
|
||||
cl_assert_write_object(source, data, OBJ_BLOB, &written_oid);
|
||||
|
||||
cl_must_pass(odb_source_read_object_stream(&stream, &source->base,
|
||||
&written_oid));
|
||||
|
|
@ -141,8 +151,7 @@ void test_odb_inmemory__for_each_object(void)
|
|||
strbuf_reset(&buf);
|
||||
strbuf_addf(&buf, "%d", i);
|
||||
|
||||
cl_must_pass(odb_source_write_object(&source->base, buf.buf, buf.len,
|
||||
OBJ_BLOB, &written_oid, NULL, 0));
|
||||
cl_assert_write_object(source, buf.buf, OBJ_BLOB, &written_oid);
|
||||
cl_must_pass(oidset_insert(&expected_oids, &written_oid));
|
||||
}
|
||||
|
||||
|
|
@ -174,12 +183,9 @@ void test_odb_inmemory__for_each_object_can_abort_iteration(void)
|
|||
struct object_id written_oid;
|
||||
unsigned counter = 0;
|
||||
|
||||
cl_must_pass(odb_source_write_object(&source->base, "1", 1,
|
||||
OBJ_BLOB, &written_oid, NULL, 0));
|
||||
cl_must_pass(odb_source_write_object(&source->base, "2", 1,
|
||||
OBJ_BLOB, &written_oid, NULL, 0));
|
||||
cl_must_pass(odb_source_write_object(&source->base, "3", 1,
|
||||
OBJ_BLOB, &written_oid, NULL, 0));
|
||||
cl_assert_write_object(source, "1", OBJ_BLOB, &written_oid);
|
||||
cl_assert_write_object(source, "2", OBJ_BLOB, &written_oid);
|
||||
cl_assert_write_object(source, "3", OBJ_BLOB, &written_oid);
|
||||
|
||||
cl_assert_equal_i(odb_source_for_each_object(&source->base, NULL,
|
||||
abort_after_two_objects,
|
||||
|
|
@ -199,12 +205,9 @@ void test_odb_inmemory__count_objects(void)
|
|||
cl_must_pass(odb_source_count_objects(&source->base, 0, &count));
|
||||
cl_assert_equal_u(count, 0);
|
||||
|
||||
cl_must_pass(odb_source_write_object(&source->base, "1", 1,
|
||||
OBJ_BLOB, &written_oid, NULL, 0));
|
||||
cl_must_pass(odb_source_write_object(&source->base, "2", 1,
|
||||
OBJ_BLOB, &written_oid, NULL, 0));
|
||||
cl_must_pass(odb_source_write_object(&source->base, "3", 1,
|
||||
OBJ_BLOB, &written_oid, NULL, 0));
|
||||
cl_assert_write_object(source, "1", OBJ_BLOB, &written_oid);
|
||||
cl_assert_write_object(source, "2", OBJ_BLOB, &written_oid);
|
||||
cl_assert_write_object(source, "3", OBJ_BLOB, &written_oid);
|
||||
|
||||
cl_must_pass(odb_source_count_objects(&source->base, 0, &count));
|
||||
cl_assert_equal_u(count, 3);
|
||||
|
|
@ -228,8 +231,7 @@ void test_odb_inmemory__find_abbrev_len(void)
|
|||
*
|
||||
* With only one blob written we expect a length of 4.
|
||||
*/
|
||||
cl_must_pass(odb_source_write_object(&source->base, "368317", strlen("368317"),
|
||||
OBJ_BLOB, &oid1, NULL, 0));
|
||||
cl_assert_write_object(source, "368317", OBJ_BLOB, &oid1);
|
||||
cl_must_pass(odb_source_find_abbrev_len(&source->base, &oid1, 4,
|
||||
&abbrev_len));
|
||||
cl_assert_equal_u(abbrev_len, 4);
|
||||
|
|
@ -238,8 +240,7 @@ void test_odb_inmemory__find_abbrev_len(void)
|
|||
* With both objects present, the shared 10-character prefix means we
|
||||
* need at least 11 characters to uniquely identify either object.
|
||||
*/
|
||||
cl_must_pass(odb_source_write_object(&source->base, "514796", strlen("514796"),
|
||||
OBJ_BLOB, &oid2, NULL, 0));
|
||||
cl_assert_write_object(source, "514796", OBJ_BLOB, &oid2);
|
||||
cl_must_pass(odb_source_find_abbrev_len(&source->base, &oid1, 4,
|
||||
&abbrev_len));
|
||||
cl_assert_equal_u(abbrev_len, 11);
|
||||
|
|
@ -255,13 +256,11 @@ void test_odb_inmemory__freshen_object(void)
|
|||
const char *end;
|
||||
|
||||
cl_must_pass(parse_oid_hex_algop(RANDOM_OID, &oid, &end, repo.hash_algo));
|
||||
cl_assert_equal_i(odb_source_freshen_object(&source->base, &oid), 0);
|
||||
cl_assert_equal_i(odb_source_freshen_object(&source->base, &oid, NULL), 0);
|
||||
|
||||
cl_must_pass(odb_source_write_object(&source->base, "foobar",
|
||||
strlen("foobar"), OBJ_BLOB,
|
||||
&written_oid, NULL, 0));
|
||||
cl_assert_write_object(source, "foobar", OBJ_BLOB, &written_oid);
|
||||
cl_assert_equal_i(odb_source_freshen_object(&source->base,
|
||||
&written_oid), 1);
|
||||
&written_oid, NULL), 1);
|
||||
|
||||
odb_source_free(&source->base);
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue