/* archive-pfs.c: * * vim:smartindent ts=8:sts=2:sta:et:ai:shiftwidth=2 **************************************************************** * Copyright (C) 2003 Tom Lord * * See the file "COPYING" for further information about * the copyright and warranty status of this work. */ #include "po/gettext.h" #include "hackerlab/bugs/exception.h" #include "hackerlab/bugs/panic.h" #include "hackerlab/os/errno.h" #include "hackerlab/os/errno-to-string.h" #include "hackerlab/mem/mem.h" #include "hackerlab/mem/talloc.h" #include "hackerlab/char/str.h" #include "hackerlab/hash/md5.h" #include "hackerlab/hash/sha1.h" #include "hackerlab/fs/file-names.h" #include "hackerlab/fs/cwd.h" #include "hackerlab/vu/safe.h" #include "libawk/relational.h" #include "libfsutils/dir-listing.h" #include "libfsutils/file-contents.h" #include "libfsutils/copy-file.h" #include "libfsutils/read-line.h" #include "libfsutils/rmrf.h" #include "libfsutils/tmp-files.h" #include "libarch/chatter.h" #include "libarch/namespace.h" #include "libarch/archives.h" #include "libarch/archive-version.h" #include "libarch/pfs.h" #include "libarch/archive-pfs.h" #include "libarch/pfs-signatures.h" #include "libarch/exec.h" /* __STDC__ prototypes for static functions */ static int pfs_close (void * arch_archive_void); static t_uchar * pfs_archive_version (struct arch_archive * a); static rel_table pfs_categories (struct arch_archive * a); static rel_table pfs_branches (struct arch_archive * a, t_uchar * category); static rel_table pfs_versions (struct arch_archive * a, t_uchar * package); static rel_table pfs_revisions (struct arch_archive * a, t_uchar * version); static t_uchar * pfs_archive_log (struct arch_archive * a, t_uchar * revision); static int pfs_revision_type (enum arch_revision_type * type, int * archive_cached, int *has_ancestry, struct arch_archive * a, t_uchar const * revision); static int list_contains (rel_table files, t_uchar * name); static void pfs_get_patch (int out_fd, struct arch_archive * a, t_uchar * revision); static void pfs_get_cached (int out_fd, struct arch_archive * a, t_uchar * revision); static void pfs_get_import (int out_fd, struct arch_archive * a, t_uchar * revision); static t_uchar * pfs_get_continuation (struct arch_archive * a, t_uchar * revision); static int pfs_set_meta_info (struct arch_archive * a, t_uchar * meta_info_name, t_uchar * meta_info_value); static t_uchar * pfs_get_meta_info (struct arch_archive * a, t_uchar * meta_info_name); static int pfs_make_category (t_uchar ** errstr, struct arch_archive * a, t_uchar * category); static int pfs_make_branch (t_uchar ** errstr, struct arch_archive * a, t_uchar * branch); static int pfs_make_version (t_uchar ** errstr, struct arch_archive * a, t_uchar * version); static int pfs_lock_revision (t_uchar ** errstr, struct arch_archive * a, t_uchar * version, t_uchar * prev_level, t_uchar * uid, t_uchar * txn_id, t_uchar * new_level); static int pfs_revision_ready (t_uchar ** errstr, struct arch_archive * a, struct arch_archive * from_archive, t_uchar * version, t_uchar * prev_level, t_uchar * uid, t_uchar * txn_id, t_uchar * new_level); static int pfs_finish_revision (t_uchar ** errstr, struct arch_archive * a, t_uchar * version, t_uchar * prev_level, t_uchar * uid, t_uchar * txn_id, t_uchar * new_level); static enum arch_revision_lock_state pfs_lock_state (t_uchar ** prev_level_ret, t_uchar ** uid_ret, t_uchar ** txn_id_ret, struct arch_archive * a, t_uchar * version); static int pfs_break_revision_lock (t_uchar ** errstr, struct arch_archive * a, t_uchar * version, t_uchar * prev_level, t_uchar * uid, t_uchar * txn_id); static void safely_remove_stale_broken_lock_dir (struct arch_pfs_archive * arch, t_uchar * broken_dir, t_uchar * prev_level); static void safely_remove_spent_lock (struct arch_pfs_archive * arch, t_uchar * spent_lock); static int pfs_put_log (t_uchar ** errstr, struct arch_archive * a, t_uchar * version, t_uchar * prev_level, t_uchar * uid, t_uchar * txn_id, t_uchar * log_text); static int pfs_put_continuation (t_uchar ** errstr, struct arch_archive * a, t_uchar * version, t_uchar * prev_level, t_uchar * uid, t_uchar * txn_id, t_uchar * continuation); static int pfs_put_changeset (t_uchar ** errstr, struct arch_archive * a, t_uchar * version, t_uchar * prev_level, t_uchar * uid, t_uchar * txn_id, t_uchar * level, int in_fd); static int pfs_put_import (t_uchar ** errstr, struct arch_archive * a, t_uchar * version, t_uchar * prev_level, t_uchar * uid, t_uchar * txn_id, t_uchar * level, int in_fd); static int pfs_put_cached (t_uchar ** errstr, struct arch_archive * a, struct arch_archive *from_archive, t_uchar * revision, int in_fd); static int pfs_delete_cached (t_uchar ** errstr, struct arch_archive * a, t_uchar * revision); static void pfs_repair_non_txnal (int chatter_fd, struct arch_archive * a); static void pfs_get_ancestry (int out_fd, struct arch_archive * a, t_uchar * revision); static int pfs_put_ancestry (t_uchar **errstr, struct arch_archive *a, struct arch_archive *from_archive, t_uchar * revision, int in_fd); static int pfs_delete_ancestry (t_uchar ** errstr, struct arch_archive * a, t_uchar * revision); static void sign_and_upload (struct arch_archive * a, t_uchar * tmp_file, t_uchar * path, int in_fd, t_uchar * description, int batteries_to_power); static void pfs_create_signature_file (struct arch_archive * a, t_uchar * revision); static int pfs_finish_signature_file (struct arch_archive *a, struct arch_archive *from_archive, t_uchar * version, t_uchar * prev_level, t_uchar * uid, t_uchar * txn_id, t_uchar * new_level); static int pfs_finish_signature_file_worker (struct arch_archive * a, struct arch_archive *from_archive, t_uchar * revision, t_uchar * dir, t_uchar * checksumname); static void insert_sums (struct arch_pfs_archive *arch, t_uchar * description, t_uchar * data_file); static void dir_changed (struct arch_pfs_archive * arch, t_uchar const *dir); static int pfs_delete_file_pair (t_uchar ** errstr, struct arch_archive * a, t_uchar * file1, t_uchar *file2); static int pfs_set_mirror (t_uchar ** errstr, struct arch_archive * archive, int enabled); static void pfs_clear_cached (struct arch_pfs_archive * arch); static struct arch_archive_vtable pfs_vtable = { "pfs", pfs_archive_version, pfs_categories, pfs_branches, pfs_versions, pfs_revisions, pfs_archive_log, pfs_revision_type, pfs_get_patch, pfs_get_cached, pfs_get_import, pfs_get_continuation, pfs_set_meta_info, pfs_get_meta_info, pfs_make_category, pfs_make_branch, pfs_make_version, pfs_lock_revision, pfs_revision_ready, pfs_finish_revision, pfs_break_revision_lock, pfs_lock_state, pfs_put_log, pfs_put_continuation, pfs_put_changeset, pfs_put_import, pfs_put_cached, pfs_delete_cached, pfs_repair_non_txnal, pfs_get_ancestry, pfs_put_ancestry, pfs_delete_ancestry, pfs_set_mirror }; void arch_pfs_make_archive (t_uchar * name, t_uchar *location, t_uchar * version, t_uchar * mirror_of, int dot_listing_lossage, int signed_archive) { arch_pfs_pfs_make_archive (name, location, version, mirror_of, dot_listing_lossage, signed_archive); } int arch_pfs_archive_connect (struct arch_archive ** a, int soft_errors) { struct arch_pfs_archive * arch = (struct arch_pfs_archive *)*a; arch = talloc_realloc_size (0, arch, sizeof (struct arch_pfs_archive)); talloc_set_destructor (arch, pfs_close); *a = (struct arch_archive *)arch; arch->arch.vtable = &pfs_vtable; arch->from_arch = 0; arch->txn_signature_file = NULL; arch->txn_signature_fd = -1; arch->root_dirs = NULL; arch->pfs = arch_pfs_connect (arch->arch.location, soft_errors); if (!arch->pfs) { if (soft_errors) { return -1; } panic ("failed to connect to archive with soft errors zero"); } return 0; } /** * \brief see arch_archive_close */ int pfs_close (void * arch_archive_void) { struct arch_archive *a = (struct arch_archive *) arch_archive_void; struct arch_pfs_archive * arch = (struct arch_pfs_archive *)a; if (arch->pfs) arch_pfs_disconnect (&arch->pfs); pfs_clear_cached (arch); return arch_archive_finalise (a); } /** * \brief see arch_archive.archive_version */ static t_uchar * pfs_archive_version (struct arch_archive * a) { struct arch_pfs_archive * arch = (struct arch_pfs_archive *)a; t_uchar * path = 0; t_uchar * meta_info_path = 0; t_uchar * contents = NULL; t_uchar * nl; path = arch_fs_archive_archive_version_path (0); meta_info_path = arch_fs_archive_meta_info_path (0); if (arch_pfs_file_exists (arch->pfs, path)) contents = arch_pfs_file_contents (arch->pfs, path, 0); else if (arch_pfs_file_exists (arch->pfs, meta_info_path)) { /* most likely a mirror that's missing a .archive-version file */ safe_printfmt (2, _("Guessing archive format as tla 1.0. This is normal with old archives, you can ignore it unless other errors occur.\n")); contents = str_save (0, arch_tree_format_1_str); } else { goto done; } nl = str_chr_index (contents, '\n'); if (nl) { size_t len; len = nl - contents + 1; contents = lim_realloc (0, contents, len); contents[len - 1] = 0; } done: lim_free (0, path); lim_free (0, meta_info_path); return contents; } static rel_table pfs_baz_search (struct arch_archive *a, t_uchar * prefix, enum arch_valid_package_name_types req_type, enum arch_parse_package_name_type ret_type) { struct arch_pfs_archive * arch = (struct arch_pfs_archive *)a; rel_table answer = 0; int x; if (!arch->root_dirs) { arch->root_dirs = arch_pfs_directory_files (arch->pfs, ".", 0); rel_sort_table_by_field (0, arch->root_dirs, 0); } for (x = 0; x < rel_n_records (arch->root_dirs); ++x) { if (arch_valid_package_name (arch->root_dirs[x][0], arch_no_archive, req_type, 1) && !str_cmp_prefix(prefix, arch->root_dirs[x][0])) rel_add_records (&answer, rel_make_record (arch_parse_package_name(ret_type, NULL, arch->root_dirs[x][0]), 0), 0); } rel_uniq_by_field (&answer, 0); return answer; } static rel_table pfs_categories (struct arch_archive * a) { struct arch_pfs_archive * arch = (struct arch_pfs_archive *)a; rel_table files = 0; rel_table answer = 0; if (a->type == arch_archive_baz) return pfs_baz_search (a, NULL, arch_req_category, arch_ret_category); files = arch_pfs_directory_files (arch->pfs, ".", 0); answer = arch_pick_categories_by_field (files, 0); rel_free_table (files); return answer; } static rel_table pfs_branches (struct arch_archive * a, t_uchar * category) { struct arch_pfs_archive * arch = (struct arch_pfs_archive *)a; t_uchar * category_path = 0; rel_table files = 0; rel_table answer = 0; t_uchar *search_prefix = str_alloc_cat_many (0, category, "--", str_end); if (a->type == arch_archive_baz) return pfs_baz_search (a, search_prefix, arch_req_package, arch_ret_package); category_path = arch_fs_archive_category_path (a, 0, category); files = arch_pfs_directory_files (arch->pfs, category_path, 1); answer = arch_pick_branches_by_field (files, 0); rel_free_table (files); lim_free (0, search_prefix); lim_free (0, category_path); return answer; } static rel_table pfs_versions (struct arch_archive * a, t_uchar * package) { struct arch_pfs_archive * arch = (struct arch_pfs_archive *)a; t_uchar * branch_path = 0; rel_table files = 0; rel_table answer = 0; t_uchar *search_prefix = str_alloc_cat_many (0, package, "--", str_end); if (a->type == arch_archive_baz) return pfs_baz_search (a, search_prefix, arch_req_version, arch_ret_package_version); branch_path = arch_fs_archive_branch_path (a, 0, package); files = arch_pfs_directory_files (arch->pfs, branch_path, 1); answer = arch_pick_versions_by_field (files, 0); rel_free_table (files); lim_free (0, branch_path); lim_free (0, search_prefix); return answer; } static rel_table pfs_revisions (struct arch_archive * a, t_uchar * version) { struct arch_pfs_archive * arch = (struct arch_pfs_archive *)a; t_uchar * version_path = 0; rel_table files = 0; rel_table answer = 0; version_path = arch_fs_archive_version_path (a, 0, version); files = arch_pfs_directory_files (arch->pfs, version_path, 0); answer = arch_pick_patch_levels_by_field (files, 0); rel_free_table (files); lim_free (0, version_path); return answer; } static t_uchar * pfs_archive_log (struct arch_archive * a, t_uchar * revision) { struct arch_pfs_archive * arch = (struct arch_pfs_archive *)a; t_uchar * revision_log_path = 0; t_uchar * answer = 0; if (arch_pfs_ensure_checksum_data (arch, revision)) { safe_printfmt (2, "trouble reading checksum file for %s/%s url: %s\n", a->official_name, revision, a->location); exit (2); } revision_log_path = arch_fs_archive_revision_log_path (a, 0, revision); answer = arch_pfs_checked_file_contents (arch, revision, revision_log_path); lim_free (0, revision_log_path); return answer; } static t_uchar * pfs_file_path_to_tail (t_uchar *file_path) { t_uchar *result = file_name_tail (0, file_path); lim_free (0, file_path); return result; } static int pfs_revision_type (enum arch_revision_type * type, int * archive_cached, int *has_ancestry, struct arch_archive * a, t_uchar const * revision) { struct arch_pfs_archive * arch = (struct arch_pfs_archive *)a; t_uchar * continuation_path = 0; t_uchar * cacherev_path = 0; t_uchar * log_path = 0; t_uchar * continuation_tail = 0; t_uchar * changeset_tail = 0; t_uchar * import_tail = 0; t_uchar * cacherev_tail = 0; t_uchar * log_tail = 0; t_uchar * revision_dir = 0; t_uchar * ancestry_tail = 0; rel_table files = 0; int has_continuation_file; int has_changeset_file; int has_import_file; int has_cacherev_file; int has_log_file; int has_ancestry_file; int result = 0; continuation_path = arch_fs_archive_continuation_path (a, 0, revision); cacherev_path = arch_fs_archive_cached_path (a, 0, revision); log_path = arch_fs_archive_revision_log_path (a, 0, revision); continuation_tail = file_name_tail (0, continuation_path); changeset_tail = pfs_file_path_to_tail (arch_fs_archive_changeset_path (a, 0, revision)); import_tail = pfs_file_path_to_tail (arch_fs_archive_import_path (a, 0, revision)); cacherev_tail = file_name_tail (0, cacherev_path); log_tail = file_name_tail (0, log_path); ancestry_tail = pfs_file_path_to_tail (arch_fs_archive_ancestry_path (a, 0, revision)); revision_dir = file_name_directory_file (0, continuation_path); files = arch_pfs_directory_files (arch->pfs, revision_dir, 1); if (files == 0) { result = -1; goto pfs_revision_type_cleanup; } if (arch_pfs_ensure_checksum_data (arch, revision)) { safe_printfmt (2, "trouble reading checksum file for %s/%s url: %s\n", a->official_name, revision, a->location); exit (2); } has_continuation_file = list_contains (files, continuation_tail); has_changeset_file = list_contains (files, changeset_tail); has_import_file = list_contains (files, import_tail); has_cacherev_file = list_contains (files, cacherev_tail); has_log_file = list_contains (files, log_tail); has_ancestry_file = list_contains (files, ancestry_tail); if (arch_pfs_checksum_governs (arch, revision) && ((!!has_continuation_file != !!arch_pfs_checksum_anticipates_file (a, revision, continuation_tail)) || (!!has_changeset_file != !!arch_pfs_checksum_anticipates_file (a, revision, changeset_tail)) || (!!has_import_file != !!arch_pfs_checksum_anticipates_file (a, revision, import_tail)) || (!!has_cacherev_file != !!arch_pfs_checksum_anticipates_file (a, revision, cacherev_tail)) || (!!has_log_file != !!arch_pfs_checksum_anticipates_file (a, revision, log_tail)) || (!!has_ancestry_file != !!arch_pfs_checksum_anticipates_file (a, revision, ancestry_tail)) )) { safe_printfmt (2, ("\n" "***********************************\n" "\n" " CHECKSUM FILE(S) DISAGREE WITH\n" " DIRECTORY LISTING ABOUT WHAT\n" " FILES SHOULD BE PRESENT IN\n" " REVISION DIR OF ARCHIVE\n" "\n" " archive: %s\n" " revision: %s\n" " url: %s\n" "\n" "***********************************\n" "\n"), a->official_name, revision, a->location); if (arch_pfs_checksum_governs_strictly (arch)) exit (2); } if (list_contains (files, continuation_tail)) { if (type) *type = arch_continuation_revision; } else if (list_contains (files, changeset_tail)) { if (type) *type = arch_simple_revision; } else if (list_contains (files, import_tail)) { if (type) *type = arch_import_revision; } else { result= -1; goto pfs_revision_type_cleanup; } if (archive_cached) *archive_cached = has_cacherev_file; if (has_ancestry) *has_ancestry = has_ancestry_file; pfs_revision_type_cleanup: lim_free (0, continuation_path); lim_free (0, cacherev_path); lim_free (0, log_path); lim_free (0, continuation_tail); lim_free (0, changeset_tail); lim_free (0, import_tail); lim_free (0, cacherev_tail); lim_free (0, log_tail); lim_free (0, ancestry_tail); lim_free (0, revision_dir); rel_free_table (files); return result; } static int list_contains (rel_table files, t_uchar * name) { int x; for (x = 0; x < rel_n_records (files); ++x) { if (!str_cmp (files[x][0], name)) return 1; } return 0; } static void pfs_get_patch (int out_fd, struct arch_archive * a, t_uchar * revision) { struct arch_pfs_archive * arch = (struct arch_pfs_archive *)a; t_uchar * changeset_path = 0; if (arch_pfs_ensure_checksum_data (arch, revision)) { safe_printfmt (2, "trouble reading checksum file for %s/%s url: %s\n", a->official_name, revision, a->location); exit (2); } changeset_path = arch_fs_archive_changeset_path (a, 0, revision); arch_pfs_checked_get_file (arch, revision, out_fd, changeset_path); lim_free (0, changeset_path); } static void pfs_get_cached (int out_fd, struct arch_archive * a, t_uchar * revision) { struct arch_pfs_archive * arch = (struct arch_pfs_archive *)a; t_uchar * cached_path = 0; if (arch_pfs_ensure_checksum_data (arch, revision)) { safe_printfmt (2, "trouble reading checksum file for %s/%s url: %s\n", a->official_name, revision, a->location); exit (2); } cached_path = arch_fs_archive_cached_path (a, 0, revision); arch_pfs_checked_get_file (arch, revision, out_fd, cached_path); lim_free (0, cached_path); } static void pfs_get_import (int out_fd, struct arch_archive * a, t_uchar * revision) { struct arch_pfs_archive * arch = (struct arch_pfs_archive *)a; t_uchar * import_path = 0; if (arch_pfs_ensure_checksum_data (arch, revision)) { safe_printfmt (2, "trouble reading checksum file for %s/%s url: %s\n", a->official_name, revision, a->location); exit (2); } import_path = arch_fs_archive_import_path (a, 0, revision); arch_pfs_checked_get_file (arch, revision, out_fd, import_path); lim_free (0, import_path); } static t_uchar * pfs_get_continuation (struct arch_archive * a, t_uchar * revision) { struct arch_pfs_archive * arch = (struct arch_pfs_archive *)a; t_uchar * continuation_path = 0; t_uchar * answer = 0; if (arch_pfs_ensure_checksum_data (arch, revision)) { safe_printfmt (2, "trouble reading checksum file for %s/%s url: %s\n", a->official_name, revision, a->location); exit (2); } continuation_path = arch_fs_archive_continuation_path (a, 0, revision); answer = arch_pfs_checked_file_contents (arch, revision, continuation_path); lim_free (0, continuation_path); return answer; } static int pfs_set_meta_info (struct arch_archive * a, t_uchar * meta_info_name, t_uchar * meta_info_value) { struct arch_pfs_archive * arch = (struct arch_pfs_archive *)a; t_uchar * meta_info_path = arch_fs_archive_meta_info_item_path (0, meta_info_name); int answer; if (meta_info_value == NULL) { answer = arch_pfs_rm (arch->pfs, meta_info_path, 1); } else { answer = arch_pfs_set_file_contents (arch->pfs, meta_info_path, meta_info_value, 1); } lim_free (0, meta_info_path); return answer; } static t_uchar * pfs_get_meta_info (struct arch_archive * a, t_uchar * meta_info_name) { struct arch_pfs_archive * arch = (struct arch_pfs_archive *)a; t_uchar * meta_info_path = 0; t_uchar * answer = 0; meta_info_path = arch_fs_archive_meta_info_item_path (0, meta_info_name); answer = arch_pfs_file_contents (arch->pfs, meta_info_path, 1); lim_free (0, meta_info_path); return answer; } static int pfs_make_category (t_uchar ** errstr, struct arch_archive * a, t_uchar * category) { struct arch_pfs_archive * arch = (struct arch_pfs_archive *)a; t_uchar * cat_path = 0; int result; result = -1; if (errstr) *errstr = "internal error in archive-pfs.c(pfs_make_category)"; invariant (arch_valid_package_name (category, arch_no_archive, arch_req_category, 0)); cat_path = arch_fs_archive_category_path (a, 0, category); if (arch_pfs_file_exists (arch->pfs, cat_path)) { if (errstr) *errstr = "category already exists"; result = -1; } else { t_uchar * cat_path_dir = 0; t_uchar * cat_tmp_path = 0; /* Why mkdir/rename? * * Because the archive might exist on NFS on which mkdir can * succeed for more than one client. * * Rename to a common target can succeed for more than one NFS * client as well, but not if each client uses a unique * source for the rename. */ cat_path_dir = file_name_directory_file (0, cat_path); cat_tmp_path = archive_tmp_file_name (cat_path_dir, ",,new-category"); arch_pfs_mkdir (arch->pfs, cat_tmp_path, 0777, 0); if (!arch_pfs_rename (arch->pfs, errstr, cat_tmp_path, cat_path, 1)) { result = 0; if (errstr) *errstr = 0; } else { result = -1; } if (arch->arch.http_blows) { arch_pfs_update_listing_file (arch->pfs, cat_path); arch_pfs_update_root_listing_file (arch->pfs, "."); } lim_free (0, cat_tmp_path); lim_free (0, cat_path_dir); } lim_free (0, cat_path); return result; } static int pfs_make_branch (t_uchar ** errstr, struct arch_archive * a, t_uchar * branch) { struct arch_pfs_archive * arch = (struct arch_pfs_archive *)a; t_uchar * branch_path = 0; int result = -1; if (errstr) *errstr = "internal error in archive-pfs.c(pfs_make_branch)"; invariant (arch_valid_package_name (branch, arch_no_archive, arch_req_package, 0)); branch_path = arch_fs_archive_branch_path (a, 0, branch); if (arch_pfs_file_exists (arch->pfs, branch_path)) { if (errstr) *errstr = "branch already exists"; result = -1; } else { t_uchar * branch_path_dir = 0; t_uchar * branch_tmp_path = 0; branch_path_dir = file_name_directory_file (0, branch_path); branch_tmp_path = archive_tmp_file_name (branch_path_dir, ",,new-branch"); arch_pfs_mkdir (arch->pfs, branch_tmp_path, 0777, 0); if (!arch_pfs_rename (arch->pfs, errstr, branch_tmp_path, branch_path, 1)) { result = 0; if (errstr) *errstr = 0; } else { result = -1; } if (arch->arch.http_blows) { arch_pfs_update_listing_file (arch->pfs, branch_path); arch_pfs_update_listing_file (arch->pfs, branch_path_dir); } lim_free (0, branch_tmp_path); lim_free (0, branch_path_dir); } lim_free (0, branch_path); return result; } static int pfs_make_version (t_uchar ** errstr, struct arch_archive * a, t_uchar * version) { struct arch_pfs_archive * arch = (struct arch_pfs_archive *)a; t_uchar * version_path = 0; int result = -1; if (errstr) *errstr = "internal error in archive-pfs.c(pfs_make_version)"; invariant (arch_valid_package_name (version, arch_no_archive, arch_req_version, 0)); version_path = arch_fs_archive_version_path (a, 0, version); /* TODO check via root_dirs for speed ? */ if (arch_pfs_file_exists (arch->pfs, version_path)) { if (errstr) *errstr = "version already exists"; result = -1; } else { t_uchar * version_path_dir = 0; t_uchar * version_tmp_path = 0; t_uchar * lock_dir_path = 0; t_uchar * contents_dir_path = 0; version_path_dir = file_name_directory_file (0, version_path); version_tmp_path = archive_tmp_file_name (version_path_dir, ",,new-branch"); lock_dir_path = file_name_in_vicinity (0, version_tmp_path, "++revision-lock"); contents_dir_path = file_name_in_vicinity (0, lock_dir_path, "+contents"); arch_pfs_mkdir (arch->pfs, version_tmp_path, 0777, 0); arch_pfs_mkdir (arch->pfs, lock_dir_path, 0777, 0); arch_pfs_mkdir (arch->pfs, contents_dir_path, 0777, 0); if (!arch_pfs_rename (arch->pfs, errstr, version_tmp_path, version_path, 1)) { result = 0; if (errstr) *errstr = 0; } else { result = -1; } dir_changed (arch, version_path); if (arch->arch.http_blows) { if (version_path_dir) arch_pfs_update_listing_file (arch->pfs, version_path_dir); else arch_pfs_update_root_listing_file (arch->pfs, "."); } lim_free (0, version_tmp_path); lim_free (0, version_path_dir); lim_free (0, lock_dir_path); lim_free (0, contents_dir_path); pfs_clear_cached(arch); } lim_free (0, version_path); return result; } /**************************************************************** * Revision Locks * * A version contains a a sequence of revisions. Each new revision * must be created in-sequence, in an atomic, isolated, and durable * event. * * In file system archives, a version is a directory, and each revision * a subdirectory with a patch level for a name (base-0, patch-1, ...) * * The ordinary rename system call almost gives us a mechanism for this: * a client could mkdir ,,wants-to-be-patch-1-txnid, fill that with * the revisions data, then rename it to patch-1. We could trust clients * not to create patch-N unless patch-(N-1) already exists, and the rename * will fail if a concurrent client succeeds at committing patch-N first. * * Alas, theres a catch here. The "revision after" base-0 or any * patch-N does not have a fixed name. It might be patch-(N+1), or it * might be "version-0". It would be disasterous if two concurrent clients * simultaneously created both patch-(N+1) and version-0 as successors * to patch-N (or base-0). rename doesn't directly solve this problem. * * (Additionally, we want to support "persistent locks" -- to allow a * user to claim the lock for a new revision prior to actually committing it, * in order to impose an advisory barrier to anybody else trying to commit * that revision.) * * So, here's how it works instead: * * Every subtree for a given version contains, at all times, exactly * one (non-nested) "revision-lock directory". That directory always * has a subdirectory called "+contents". In the normal course of * things, the "+contents" directory will eventually be renamed to * become the new revision directory. * * In the general, the name of the revision-lock directory indicates * the state of the lock. When a write transaction holds the lock, * it fills "+contents" with the data for the new revision (including * a new, nested revision lock directory, then renames "+contents" * to patch-N (or version-0 or base-0 or versionfix-N), and cleans up. * * * Possible Revision Lock States: * * [A] No revisions exist yet, lock is unlocked in the usual way: * * version/ * version/+revision-lock == $lock * version/+revision-lock/+contents * * * [B] Patch level LVL is the most recent, lock is unlocked in the usual way: * * version/ * version/LVL * version/LVL/+revision-lock == $lock * version/LVL/+revision-lock/+contents (empty) * * * In the rest, if no revisions yet exist, then LVL is "absolute-0" * * [C] Lock was recently broken, or a lock break operation was interrupted * after breaking the lock, but before cleaning up: * * version/ * version/+revision-lock-broken--LVL == $broken_dir * version/+revision-lock-broken--LVL/,,remade-lock--LVL == $broken (the lock itself) * version/+revision-lock-broken--LVL/,,remade-lock--LVL/+contents * version/+revision-lock-broken--LVL/... junk from failed transaction * * * [D] Persistent lock is currently held by user UID * * version/ * version/+revision-lock-held--LVL--UID == $locked * version/+revision-lock-held--LVL--UID/+contents (empty) * * * [E] Lock is currently held by user UID for client process CLIENT * * version/ * version/+revision-lock-held--LVL--UID.CLIENT == $locked * version/+revision-lock-held--LVL--UID/+contents * version/+revision-lock-held--LVL--UID/+contents/... (new revision data) * * * [-] Junk that can be left around when cleanups dont finish (safe to delete * things marked with "!"): * * version/ * ! version/+revision-lock-held--LVL--UID* * ! (no +contents subdir, not necessarilly empty, otherwise) * * version/ * ! version/+revision-lock-broken--LVL * ! (no ,,remade-lock--LVL subdir, not necessarilly empty, otherwise) * * version/ * version/revision--(LVL + 1) i.e., completed next revision * ! version/revision--(LVL + 1)/++version-lock/,* failed attempt to break lock * */ static int pfs_lock_revision (t_uchar ** errstr, struct arch_archive * a, t_uchar * version, t_uchar * prev_level, t_uchar * uid, t_uchar * txn_id, t_uchar * new_level) { struct arch_pfs_archive * arch = (struct arch_pfs_archive *)a; t_uchar * unlocked_path = 0; t_uchar * persistent_locked_path = 0; t_uchar * final_locked_path = 0; t_uchar * final_locked_contents_path = 0; t_uchar * broken_path = 0; int rename_occurred = 0; int result; result = -1; if (errstr) *errstr = "internal error in archive-pfs.c(pfs_lock_revision)"; /* txn_id == 0 * * [A] -> [D] * [B] -> [D] * [C] -> [D] * [D] -> [D] * [E] -> error (user must break lock or txn complete) * * txn_id != 0 * * [A] -> [E] * [B] -> [E] * [C] -> [E] * [D] -> [E] * [E] -> [E] (if the lock is currently held with our txn_id) * [E] -> error (otherwise) * */ unlocked_path = arch_fs_archive_revision_lock_unlocked_path (a, 0, version, prev_level); persistent_locked_path = arch_fs_archive_revision_lock_locked_path (a, 0, version, prev_level, uid, 0); final_locked_path = arch_fs_archive_revision_lock_locked_path (a, 0, version, prev_level, uid, txn_id); final_locked_contents_path = arch_fs_archive_revision_lock_locked_contents_path (a, 0, version, prev_level, uid, txn_id); broken_path = arch_fs_archive_revision_lock_broken_path (a, 0, version, prev_level); /* final_locked_path is [D] or [E] as appropriate to the txn_id argument. * * unlocked_path is [A] or [B] as appropriate to the prev_level argument. */ /* [A/B] -> [D/E] * */ if (!arch_pfs_rename (arch->pfs, errstr, unlocked_path, final_locked_path, 1)) { rename_occurred = 1; } /* [C] -> [D/E] */ else if (!arch_pfs_rename (arch->pfs, errstr, broken_path, final_locked_path, 1)) { t_uchar * broken_dir = 0; rename_occurred = 1; /* clean up the old broken lock dir */ broken_dir = file_name_directory_file (0, broken_dir); safely_remove_stale_broken_lock_dir (arch, broken_dir, prev_level); lim_free (0, broken_dir); } /* [D] -> [E] (if appropriate) */ else if (str_cmp (persistent_locked_path, final_locked_path) && !arch_pfs_rename (arch->pfs, errstr, persistent_locked_path, final_locked_path, 1)) { rename_occurred = 1; } /* [D] -> [D] or [E] -> [E] (as appropriate) */ else if (arch_pfs_file_exists (arch->pfs, final_locked_path)) { rename_occurred = 1; } /* A correctly named "++version-lock-held..." directory exists -- but is it actually * the lock? */ if (rename_occurred) { if (arch_pfs_file_exists (arch->pfs, final_locked_contents_path)) { result = 0; if (errstr) *errstr = 0; } else { result = -1; if (errstr) { *errstr = "lock held or revision already committed"; } } } lim_free (0, unlocked_path); lim_free (0, persistent_locked_path); lim_free (0, final_locked_path); lim_free (0, final_locked_contents_path); lim_free (0, broken_path); if (!result && new_level) { t_uchar * new_revision = 0; new_revision = str_alloc_cat_many (0, version, "--", new_level, str_end); pfs_create_signature_file (a, new_revision); lim_free (0, new_revision); } return result; } static int pfs_revision_ready (t_uchar ** errstr, struct arch_archive * a, struct arch_archive *from_archive, t_uchar * version, t_uchar * prev_level, t_uchar * uid, t_uchar * txn_id, t_uchar * new_level) { if (pfs_finish_signature_file (a, from_archive, version, prev_level, uid, txn_id, new_level)) { return -1; } return 0; } static int pfs_finish_revision (t_uchar ** errstr, struct arch_archive * a, t_uchar * version, t_uchar * prev_level, t_uchar * uid, t_uchar * txn_id, t_uchar * new_level) { struct arch_pfs_archive * arch = (struct arch_pfs_archive *)a; t_uchar * locked_path = 0; t_uchar * locked_contents_path = 0; t_uchar * new_lock_path = 0; t_uchar * new_contents_path = 0; t_uchar * revision = 0; t_uchar * new_rev_path = 0; int result; result = -1; if (errstr) *errstr = "internal error in archive-pfs.c(pfs_lock_revision)"; /* * [E] -> [A] if lock held with our txn_id * [E] -> error otherwise * [B,C,D] -> error */ locked_path = arch_fs_archive_revision_lock_locked_path (a, 0, version, prev_level, uid, txn_id); locked_contents_path = arch_fs_archive_revision_lock_locked_contents_path (a, 0, version, prev_level, uid, txn_id); new_lock_path = file_name_in_vicinity (0, locked_contents_path, "++revision-lock"); new_contents_path = file_name_in_vicinity (0, new_lock_path, "+contents"); revision = str_alloc_cat_many (0, version, "--", new_level, str_end); new_rev_path = arch_fs_archive_revision_path (a, 0, revision); if (!arch_pfs_file_exists (arch->pfs, locked_path)) { result = -1; *errstr = "lock not held"; } else { arch_pfs_mkdir (arch->pfs, new_lock_path, 0777, 1); arch_pfs_mkdir (arch->pfs, new_contents_path, 0777, 1); if ((1 == arch_pfs_is_dir (arch->pfs, new_contents_path)) && !arch_pfs_rename (arch->pfs, errstr, locked_contents_path, new_rev_path, 1)) { result = 0; if (errstr) *errstr = 0; safely_remove_spent_lock (arch, locked_path); if (arch->arch.http_blows) { arch_patch_id * prev_revision = 0; t_uchar * new_rev_dir_path = 0; new_rev_dir_path = file_name_directory_file (0, new_rev_path); arch_pfs_update_listing_file (arch->pfs, new_rev_path); arch_pfs_update_listing_file (arch->pfs, new_rev_dir_path); { arch_patch_id * revision_patch = arch_patch_id_new_archive (a->official_name, revision); prev_revision = arch_previous_revision (a, revision_patch); talloc_free (revision_patch); } if (prev_revision) { t_uchar * prev_rev_path = 0; prev_rev_path = arch_fs_archive_revision_path (a, 0, arch_patch_id_revision (prev_revision)); arch_pfs_update_listing_file (arch->pfs, prev_rev_path); lim_free (0, prev_rev_path); } lim_free (0, new_rev_dir_path); talloc_free (prev_revision); } } else { { safe_printfmt (2, "i/o error modifying archive (%s)\n archive: %s\n path: %s\n, url: %s\n", *errstr, arch->arch.official_name, new_contents_path, arch->arch.location); exit (2); } } } if (!result) arch_pfs_invalidate_checksum_data (arch, revision); lim_free (0, locked_path); lim_free (0, locked_contents_path); lim_free (0, new_lock_path); lim_free (0, new_contents_path); lim_free (0, revision); lim_free (0, new_rev_path); return result; } static enum arch_revision_lock_state pfs_lock_state (t_uchar ** prev_level_ret, t_uchar ** uid_ret, t_uchar ** txn_id_ret, struct arch_archive * a, t_uchar * version) { struct arch_pfs_archive * arch = (struct arch_pfs_archive *)a; rel_table revisions = 0; t_uchar * prev_level = 0; t_uchar * unlocked_path = 0; t_uchar * unlocked_contents_path = 0; t_uchar * broken_path = 0; t_uchar * broken_contents_path = 0; enum arch_revision_lock_state answer = arch_revision_unknown_lock_state; revisions = arch_archive_revisions (&arch->arch, version, 0); if (!revisions) prev_level = 0; else prev_level = str_save (0, revisions[rel_n_records (revisions) - 1][0]); if (prev_level_ret) *prev_level_ret = prev_level ? str_save (0, prev_level) : 0; unlocked_path = arch_fs_archive_revision_lock_unlocked_path (a, 0, version, prev_level); unlocked_contents_path = file_name_in_vicinity (0, unlocked_path, "+contents"); broken_path = arch_fs_archive_revision_lock_broken_path (a, 0, version, prev_level); broken_contents_path = file_name_in_vicinity (0, broken_path, "+contents"); if (uid_ret) *uid_ret = 0; if (txn_id_ret) *txn_id_ret = 0; if ((1 == arch_pfs_is_dir (arch->pfs, unlocked_contents_path)) || (1 == arch_pfs_is_dir (arch->pfs, broken_contents_path))) { answer = arch_revision_unlocked; } else { t_uchar * version_path = 0; rel_table files = 0; t_uchar * persistent_locked_path_stem = 0; int x; version_path = arch_fs_archive_version_path (a, 0, version); files = arch_pfs_directory_files (arch->pfs, version_path, 1); { t_uchar * t = 0; t = arch_fs_archive_revision_lock_locked_path (a, 0, version, prev_level, 0, 0); persistent_locked_path_stem = file_name_tail (0, t); lim_free (0, t); } for (x = 0; x < rel_n_records (files); ++x) { if (!str_cmp_prefix (persistent_locked_path_stem, files[x][0])) { t_uchar * contents_rel = 0; t_uchar * contents_path = 0; contents_rel = file_name_in_vicinity (0, files[x][0], "+contents"); contents_path = file_name_in_vicinity (0, version_path, contents_rel); if (1 != arch_pfs_is_dir (arch->pfs, contents_path)) { answer = arch_revision_illegal_lock_state; } else { t_uchar * uid_start; t_uchar * uid_end; uid_start = files[x][0] + str_length (persistent_locked_path_stem); uid_end = str_chr_rindex (uid_start, '-'); invariant (!!uid_end); --uid_end; if (uid_end < uid_start) { if (uid_ret) *uid_ret = str_save (0, uid_start); if (txn_id_ret) *txn_id_ret = 0; answer = arch_revision_user_locked; } else { t_uchar * txn_id; txn_id = uid_end + 2; if (uid_ret) *uid_ret = str_save_n (0, uid_start, uid_end - uid_start); if (txn_id_ret) *txn_id_ret = str_save (0, txn_id); answer = arch_revision_txn_locked; } } lim_free (0, contents_rel); lim_free (0, contents_path); } } lim_free (0, version_path); rel_free_table (files); lim_free (0, persistent_locked_path_stem); } rel_free_table (revisions); lim_free (0, prev_level); lim_free (0, unlocked_path); lim_free (0, unlocked_contents_path); lim_free (0, broken_path); lim_free (0, broken_contents_path); return answer; } static int pfs_break_revision_lock (t_uchar ** errstr, struct arch_archive * a, t_uchar * version, t_uchar * prev_level, t_uchar * uid, t_uchar * txn_id) { struct arch_pfs_archive * arch = (struct arch_pfs_archive *)a; t_uchar * locked_path = 0; t_uchar * locked_contents_path = 0; t_uchar * unlocked_path = 0; int result; result = -1; if (errstr) *errstr = "internal error in archive-pfs.c(pfs_break_revision_lock)"; /* * [D/E] -> [C] -> [A] if lock held with our uid/txn_id * [D/E] -> error otherwise * [A,B,C] -> error */ locked_path = arch_fs_archive_revision_lock_locked_path (a, 0, version, prev_level, uid, txn_id); locked_contents_path = arch_fs_archive_revision_lock_locked_contents_path (a, 0, version, prev_level, uid, txn_id); unlocked_path = arch_fs_archive_revision_lock_unlocked_path (a, 0, version, prev_level); if (1 != arch_pfs_is_dir (arch->pfs, locked_contents_path)) { result = -1; if (errstr) *errstr = "lock not held"; } else { t_uchar * remade_basename = 0; t_uchar * new_lock_path = 0; t_uchar * new_lock_contents_path = 0; t_uchar * version_path = 0; t_uchar * broken_lock_path = 0; t_uchar * broken_lock_dir = 0; remade_basename = str_alloc_cat (0, ",,remade-lock--", (prev_level ? prev_level : (t_uchar *)"absolute-0")); new_lock_path = file_name_in_vicinity (0, locked_contents_path, remade_basename); new_lock_contents_path = file_name_in_vicinity (0, new_lock_path, "+contents"); version_path = arch_fs_archive_version_path (a, 0, version); broken_lock_path = arch_fs_archive_revision_lock_broken_path (a, 0, version, prev_level); broken_lock_dir = file_name_directory_file (0, broken_lock_path); arch_pfs_mkdir (arch->pfs, new_lock_path, 0777, 1); arch_pfs_mkdir (arch->pfs, new_lock_contents_path, 0777, 1); if (1 != arch_pfs_is_dir (arch->pfs, new_lock_contents_path)) { result = -1; if (errstr) *errstr = "lock not held"; } else { safely_remove_stale_broken_lock_dir (arch, broken_lock_dir, prev_level); if (arch_pfs_rename (arch->pfs, 0, locked_contents_path, broken_lock_dir, 1)) { result = -1; if (errstr && !*errstr) *errstr = "lock not held"; } else { t_uchar * latest_rev = 0; /* Did the lock actually become broken (perhaps by our rename * above, or by some other process? The only other possibility * (and the only reliable way to check) is to see if, instead, * the commit completed. */ { rel_table levels = 0; levels = arch_archive_revisions (&arch->arch, version, 0); if (levels) latest_rev = str_save (0, levels[rel_n_records(levels) - 1][0]); rel_free_table (levels); } if (str_cmp (prev_level, latest_rev)) { result = -1; if (errstr) *errstr = "lock not held"; } else { arch_pfs_rename (arch->pfs, 0, broken_lock_path, unlocked_path, 1); /* We really don't care whether that rename succeeded or not. * It's just for "neatness" */ result = 0; if (errstr) *errstr = 0; } /* Note that safely_remove_stale_broken_lock_dir doesn't actually * remove the broken lock dir in the case that it isn't stale. */ safely_remove_stale_broken_lock_dir (arch, broken_lock_dir, prev_level); safely_remove_spent_lock (arch, locked_path); lim_free (0, latest_rev); } } lim_free (0, new_lock_path); lim_free (0, new_lock_contents_path); lim_free (0, version_path); lim_free (0, broken_lock_path); } lim_free (0, locked_path); lim_free (0, locked_contents_path); lim_free (0, unlocked_path); return result; } static void safely_remove_stale_broken_lock_dir (struct arch_pfs_archive * arch, t_uchar * broken_dir, t_uchar * prev_level) { t_uchar * broken_dir_tail = 0; rel_table nested = 0; int x; t_uchar * precious_basename = 0; broken_dir_tail = file_name_tail (0, broken_dir); invariant (!str_cmp_prefix ("++revision-lock-broken--", broken_dir_tail)); nested = arch_pfs_directory_files (arch->pfs, broken_dir, 1); precious_basename = str_alloc_cat (0, ",,remade-lock--", (prev_level ? prev_level : (t_uchar *)"absolute-0")); for (x = 0; x < rel_n_records (nested); ++x) { t_uchar * basename; basename = nested[x][0]; if (str_cmp (".", basename) && str_cmp ("..", basename) && str_cmp (precious_basename, basename)) { t_uchar * path = 0; path = file_name_in_vicinity (0, broken_dir, basename); arch_pfs_rmrf_file (arch->pfs, path); lim_free (0, path); } } arch_pfs_rmdir (arch->pfs, broken_dir, 1); lim_free (0, broken_dir_tail); rel_free_table (nested); lim_free (0, precious_basename); } static void safely_remove_spent_lock (struct arch_pfs_archive * arch, t_uchar * spent_lock) { t_uchar * spent_lock_tail = 0; rel_table nested = 0; t_uchar * tmp_path = 0; int x; spent_lock_tail = file_name_tail (0, spent_lock); invariant (!str_cmp_prefix ("++revision-lock-held--", spent_lock_tail)); nested = arch_pfs_directory_files (arch->pfs, spent_lock, 1); tmp_path = talloc_tmp_file_name (talloc_context, spent_lock, ",,junk"); arch_pfs_rmrf_file (arch->pfs, tmp_path); for (x = 0; x < rel_n_records (nested); ++x) { t_uchar * basename; basename = nested[x][0]; if (str_cmp (".", basename) && str_cmp ("..", basename) && str_cmp ("+contents", basename)) { t_uchar * path = 0; path = file_name_in_vicinity (0, spent_lock, basename); arch_pfs_rename (arch->pfs, 0, path, tmp_path, 1); arch_pfs_rmrf_file (arch->pfs, tmp_path); lim_free (0, path); } } arch_pfs_rmdir (arch->pfs, spent_lock, 1); lim_free (0, spent_lock_tail); rel_free_table (nested); talloc_free (tmp_path); } /**************************************************************** * Write Txn Steps */ static int pfs_put_log (t_uchar ** errstr, struct arch_archive * a, t_uchar * version, t_uchar * prev_level, t_uchar * uid, t_uchar * txn_id, t_uchar * log_text) { t_uchar * locked_contents_path = 0; t_uchar * log_path = 0; t_uchar * tmp_path = 0; int fd; invariant (!!txn_id); locked_contents_path = arch_fs_archive_revision_lock_locked_contents_path (a, 0, version, prev_level, uid, txn_id); log_path = file_name_in_vicinity (0, locked_contents_path, "log"); tmp_path = tmp_file_name_in_tmp (",,pfs-dav-put-log"); fd = safe_open (tmp_path, O_RDWR | O_CREAT | O_EXCL, 0400); safe_printfmt (fd, "%s", log_text); safe_lseek (fd, (off_t)0, SEEK_SET); sign_and_upload (a, tmp_path, log_path, fd, "log", 0); safe_unlink (tmp_path); if (errstr) *errstr = 0; safe_close (fd); lim_free (0, locked_contents_path); lim_free (0, log_path); lim_free (0, tmp_path); return 0; } static int pfs_put_continuation (t_uchar ** errstr, struct arch_archive * a, t_uchar * version, t_uchar * prev_level, t_uchar * uid, t_uchar * txn_id, t_uchar * continuation) { t_uchar * locked_contents_path = 0; t_uchar * continuation_path = 0; t_uchar * tmp_path = 0; int fd; invariant (!!txn_id); locked_contents_path = arch_fs_archive_revision_lock_locked_contents_path (a, 0, version, prev_level, uid, txn_id); continuation_path = file_name_in_vicinity (0, locked_contents_path, "CONTINUATION"); tmp_path = tmp_file_name_in_tmp (",,pfs-dav-put-log"); fd = safe_open (tmp_path, O_RDWR | O_CREAT | O_EXCL, 0400); safe_printfmt (fd, "%s", continuation); safe_lseek (fd, (off_t)0, SEEK_SET); sign_and_upload (a, tmp_path, continuation_path, fd, "CONTINUATION", 0); safe_unlink (tmp_path); if (errstr) *errstr = 0; safe_close (fd); lim_free (0, locked_contents_path); lim_free (0, continuation_path); lim_free (0, tmp_path); return 0; } static int pfs_put_changeset (t_uchar ** errstr, struct arch_archive * a, t_uchar * version, t_uchar * prev_level, t_uchar * uid, t_uchar * txn_id, t_uchar * level, int in_fd) { t_uchar * locked_contents_path = 0; t_uchar * patches = 0; t_uchar * changeset_path = 0; invariant (!!txn_id); locked_contents_path = arch_fs_archive_revision_lock_locked_contents_path (a, 0, version, prev_level, uid, txn_id); patches = str_alloc_cat_many (0, version, "--", level, ".patches.tar.gz", str_end); changeset_path = file_name_in_vicinity (0, locked_contents_path, patches); sign_and_upload (a, 0, changeset_path, in_fd, patches, 0); if (errstr) *errstr = 0; lim_free (0, locked_contents_path); lim_free (0, patches); lim_free (0, changeset_path); return 0; } static int pfs_put_import (t_uchar ** errstr, struct arch_archive * a, t_uchar * version, t_uchar * prev_level, t_uchar * uid, t_uchar * txn_id, t_uchar * level, int in_fd) { t_uchar * locked_contents_path = 0; t_uchar * import = 0; t_uchar * import_path = 0; invariant (!!txn_id); locked_contents_path = arch_fs_archive_revision_lock_locked_contents_path (a, 0, version, prev_level, uid, txn_id); import = str_alloc_cat_many (0, version, "--", level, ".src.tar.gz", str_end); import_path = file_name_in_vicinity (0, locked_contents_path, import); sign_and_upload (a, 0, import_path, in_fd, import, 0); if (errstr) *errstr = 0; lim_free (0, locked_contents_path); lim_free (0, import); lim_free (0, import_path); return 0; } static int pfs_put_cached (t_uchar ** errstr, struct arch_archive * a, struct arch_archive *from_archive, t_uchar * revision, int in_fd) { struct arch_pfs_archive * arch = (struct arch_pfs_archive *)a; t_uchar * cached_path = 0; t_uchar * cached_tail = 0; t_uchar * cached_dir = 0; cached_path = arch_fs_archive_cached_path (a, 0, revision); cached_tail = file_name_tail (0, cached_path); cached_dir = file_name_directory_file (0, cached_path); pfs_create_signature_file (a, revision); sign_and_upload (a, 0, cached_path, in_fd, cached_tail, 1); if (pfs_finish_signature_file_worker (a, from_archive, revision, cached_dir, "checksum.cacherev")) { /* FIXME: remove cacherev file. This is no worse than the fire and forget code this * block replaces.*/ exit (2); } dir_changed (arch, cached_dir); lim_free (0, cached_path); lim_free (0, cached_tail); lim_free (0, cached_dir); arch_pfs_invalidate_checksum_data (arch, revision); return 0; } int pfs_delete_file_pair (t_uchar ** errstr, struct arch_archive * a, t_uchar * file1, t_uchar *file2) { struct arch_pfs_archive * arch = (struct arch_pfs_archive *)a; if (arch_pfs_file_exists (arch->pfs, file1)) { arch_pfs_rm (arch->pfs, file1, 1); arch_pfs_rm (arch->pfs, file2, 1); if (arch->arch.http_blows) { t_uchar * dir = file_name_directory_file (0, file1); arch_pfs_update_listing_file (arch->pfs, dir); lim_free (0, dir); } } return 0; } int pfs_delete_cached (t_uchar ** errstr, struct arch_archive * a, t_uchar * revision) { t_uchar * cached_path; t_uchar * cached_checksum_path; cached_path = arch_fs_archive_cached_path (a, 0, revision); cached_checksum_path = arch_fs_archive_cached_checksum_path (a, 0, revision); pfs_delete_file_pair (errstr, a, cached_path, cached_checksum_path); lim_free (0, cached_path); lim_free (0, cached_checksum_path); return 0; } void pfs_get_ancestry (int out_fd, struct arch_archive * a, t_uchar * revision) { struct arch_pfs_archive * arch = (struct arch_pfs_archive *)a; t_uchar * ancestry_path = 0; if (arch_pfs_ensure_checksum_data (arch, revision)) { safe_printfmt (2, "trouble reading checksum file for %s/%s url: %s\n", a->official_name, revision, a->location); exit (2); } ancestry_path = arch_fs_archive_ancestry_path (a, 0, revision); arch_pfs_checked_get_file (arch, revision, out_fd, ancestry_path); lim_free (0, ancestry_path); } int pfs_put_ancestry (t_uchar **errstr, struct arch_archive *a, struct arch_archive *from_archive, t_uchar * revision, int in_fd) { struct arch_pfs_archive * arch = (struct arch_pfs_archive *)a; t_uchar * ancestry_path = 0; t_uchar * ancestry_tail = 0; t_uchar * ancestry_dir = 0; ancestry_path = arch_fs_archive_ancestry_path (a, 0, revision); ancestry_tail = file_name_tail (0, ancestry_path); ancestry_dir = file_name_directory_file (0, ancestry_path); pfs_create_signature_file (a, revision); sign_and_upload (a, 0, ancestry_path, in_fd, ancestry_tail, 1); if (pfs_finish_signature_file_worker (a, from_archive, revision, ancestry_dir, "ancestry.gz.checksum")) { /* FIXME: remove cacherev file. This is no worse than the fire and forget code this * block replaces.*/ exit (2); } dir_changed (arch, ancestry_dir); lim_free (0, ancestry_path); lim_free (0, ancestry_tail); lim_free (0, ancestry_dir); arch_pfs_invalidate_checksum_data (arch, revision); return 0; } int pfs_delete_ancestry (t_uchar ** errstr, struct arch_archive * a, t_uchar * revision) { t_uchar * ancestry_path = 0; t_uchar * ancestry_checksum_path = 0; ancestry_path = arch_fs_archive_ancestry_path (a, 0, revision); ancestry_checksum_path = arch_fs_archive_ancestry_checksum_path (a, 0, revision); pfs_delete_file_pair (errstr, a, ancestry_path, ancestry_checksum_path); lim_free (0, ancestry_path); lim_free (0, ancestry_checksum_path); return 0; } static void pfs_fixup_version (int chatter_fd, struct arch_archive *a, t_uchar * version) { t_uchar * version_path = 0; rel_table revisions = 0; int r; struct arch_pfs_archive * arch = (struct arch_pfs_archive *)a; arch_chatter (chatter_fd, "* archive-fixup: rebuilding index file for version %s\n", version); version_path = arch_fs_archive_version_path (a, 0, version); arch_pfs_update_listing_file (arch->pfs, version_path); revisions = pfs_revisions (a, version); for (r = 0; r < rel_n_records (revisions); ++r) { t_uchar * package_revision = 0; t_uchar * revision_path = 0; package_revision = str_alloc_cat_many (0, version, "--", revisions[r][0], str_end); arch_chatter (chatter_fd, "* archive-fixup: rebuilding index file for revisions %s\n", package_revision); revision_path = arch_fs_archive_revision_path (a, 0, package_revision); arch_pfs_update_listing_file (arch->pfs, revision_path); lim_free (0, package_revision); lim_free (0, revision_path); } lim_free (0, version_path); rel_free_table (revisions); } static void pfs_repair_non_txnal (int chatter_fd, struct arch_archive * a) { struct arch_pfs_archive * arch = (struct arch_pfs_archive *)a; t_uchar * meta_info_path = 0; rel_table categories = 0; int c; if (!a->http_blows) return; arch_chatter (chatter_fd, "* archive-fixup: rebuilding top level index files\n"); arch_pfs_update_root_listing_file (arch->pfs, "."); meta_info_path = arch_fs_archive_meta_info_item_path (0, "."); arch_pfs_update_listing_file (arch->pfs, meta_info_path); categories = pfs_categories (a); for (c = 0; c < rel_n_records (categories); ++c) { t_uchar * cat_path = 0; rel_table branches = 0; int b; arch_chatter (chatter_fd, "* archive-fixup: rebuilding index file for category %s\n", categories[c][0]); if (a->type != arch_archive_baz) { cat_path = arch_fs_archive_category_path (a, 0, categories[c][0]); arch_pfs_update_root_listing_file (arch->pfs, cat_path); } branches = pfs_branches (a, categories[c][0]); for (b = 0; b < rel_n_records (branches); ++b) { t_uchar * branch_path = 0; rel_table versions = 0; int v; arch_chatter (chatter_fd, "* archive-fixup: rebuilding index file for branch %s\n", branches[b][0]); if (a->type != arch_archive_baz) { branch_path = arch_fs_archive_branch_path (a, 0, branches[b][0]); arch_pfs_update_listing_file (arch->pfs, branch_path); } versions = pfs_versions (a, branches[b][0]); for (v = 0; v < rel_n_records (versions); ++v) { pfs_fixup_version (chatter_fd, a, versions[v][0]); } lim_free (0, branch_path); rel_free_table (versions); } lim_free (0, cat_path); rel_free_table (branches); } lim_free (0, meta_info_path); rel_free_table (categories); } static void sign_and_upload (struct arch_archive * a, t_uchar * tmp_file, t_uchar * path, int in_fd, t_uchar * description, int batteries_to_power) { struct arch_pfs_archive *arch = (struct arch_pfs_archive *)a; invariant (arch->txn_signature_file != 0); if (tmp_file) { insert_sums (arch, description, tmp_file); if (batteries_to_power) arch_pfs_put_atomic (arch->pfs, 0, path, 0444, in_fd, 1, 0); else arch_pfs_put_file (arch->pfs, path, 0444, in_fd, 0); } else { t_uchar * tmp_contents = 0; int my_fd = -1; tmp_contents = tmp_file_name_in_tmp (",,arch-sign-and-upload"); my_fd = safe_open (tmp_contents, O_RDWR | O_CREAT | O_EXCL, 0400); copy_fd (in_fd, my_fd); safe_lseek (my_fd, (off_t)0, SEEK_SET); insert_sums (arch, description, tmp_contents); if (batteries_to_power) arch_pfs_put_atomic (arch->pfs, 0, path, 0444, my_fd, 1, 0); else arch_pfs_put_file (arch->pfs, path, 0444, my_fd, 0); safe_close (my_fd); safe_unlink (tmp_contents); } } static void pfs_create_signature_file (struct arch_archive * a, t_uchar * revision) { struct arch_pfs_archive * arch = (struct arch_pfs_archive *)a; invariant(arch->txn_signature_file == 0); arch->txn_signature_file = tmp_file_name_in_tmp (",,archive-signature"); arch->txn_signature_fd = safe_open (arch->txn_signature_file, O_RDWR | O_CREAT | O_EXCL, 0400); arch->deferred_sha1_sums = 0; safe_printfmt (arch->txn_signature_fd, "Signature-for: %s/%s\n", a->official_name, revision); /* * file layout: * * * * * * * * * */ } static int pfs_finish_signature_file (struct arch_archive *a, struct arch_archive *from_archive, t_uchar * version, t_uchar * prev_level, t_uchar * uid, t_uchar * txn_id, t_uchar * new_level) { struct arch_pfs_archive * arch = (struct arch_pfs_archive *)a; t_uchar * dir = 0; t_uchar * revision = 0; int result = 0; invariant (!!txn_id); invariant (arch->txn_signature_file != 0); revision = str_alloc_cat_many (0, version, "--", new_level, str_end); dir = arch_fs_archive_revision_lock_locked_contents_path (a, 0, version, prev_level, uid, txn_id); if (pfs_finish_signature_file_worker (a, from_archive, revision, dir, "checksum")) { /* abort the transaction */ result = -1; } lim_free (0, dir); lim_free (0, revision); return result; } /** * \brief finish a checksum file * * sign if needed, and upload to the archive * \param a the archive the checksum is for * \param from_archive the archive that it should be copied from * \param revision name the checksum will belong to * \param dir the directory the checksum is going into * \param checksumname the filename for the checksum * \return 0 on success */ int pfs_finish_signature_file_worker (struct arch_archive * a, struct arch_archive *from_archive, t_uchar * revision, t_uchar * dir, t_uchar * checksumname) { struct arch_pfs_archive *arch = (struct arch_pfs_archive *)a; t_uchar * signed_file = 0; t_uchar * signature = 0; int error = 0; int i; signed_file = str_alloc_cat_many (0, arch->txn_signature_file, ".asc", str_end); signature = file_name_in_vicinity (0, dir, checksumname); /** * We deferred printing SHA1 sums until now, after all the MD5 sums * have been written, for backwards compatibility reasons. */ for (i = 0; i < rel_n_records (arch->deferred_sha1_sums); ++i) safe_printfmt (arch->txn_signature_fd, "sha1 %s %s\n", arch->deferred_sha1_sums[i][0], arch->deferred_sha1_sums[i][1]); rel_free_table (arch->deferred_sha1_sums); arch->deferred_sha1_sums = 0; if (a->signed_archive) { int in_fd; int signed_fd; safe_close (arch->txn_signature_fd); arch->txn_signature_fd = -1; in_fd = safe_open (arch->txn_signature_file, O_RDONLY, 0); signed_fd = safe_open (signed_file, O_RDWR | O_CREAT | O_EXCL, 0); safe_unlink (signed_file); error = arch_pfs_sign_for_archive (a, revision, checksumname, in_fd, signed_fd, from_archive); if (error) { safe_printfmt (2, "\nunable to complete transaction due to signature failure\n"); safe_unlink (arch->txn_signature_file); /* FIXME: check for resource leaks */ return -1; } safe_lseek (signed_fd, (off_t)0, SEEK_SET); arch_pfs_put_file (arch->pfs, signature, 0444, signed_fd, 0); safe_close (in_fd); safe_close (signed_fd); } else { /* TODO: check for has registry file, and if present * check warn_unsigned instead * RBC 20050309 * ->name cannot be removed here as its how the * legacy archive registration works. */ if (arch_pfs_has_signing_rule (a->registered_name)) { safe_printfmt (2, ("\n" "\n" "********************************\n" " YOU HAVE A RULE FOR SIGNING\n" " THIS ARCHIVE IN\n" " ~/.arch-params/signing/%s\n" " BUT THIS IS AN UNSIGNED ARCHIVE!\n" "\n" " archive: %s\n" "\n" "********************************\n" "\n" "\n"), a->registered_name, a->official_name); /* FIXME: check for resource leaks */ return -1; } safe_lseek (arch->txn_signature_fd, (off_t)0, SEEK_SET); arch_pfs_put_file (arch->pfs, signature, 0444, arch->txn_signature_fd, 0); safe_close (arch->txn_signature_fd); arch->txn_signature_fd = -1; } safe_unlink (arch->txn_signature_file); lim_free (0, arch->txn_signature_file); arch->txn_signature_file = 0; lim_free (0, signed_file); lim_free (0, signature); return 0; } static void insert_sums (struct arch_pfs_archive *arch, t_uchar * description, t_uchar * data_file) { md5_context_t md5c = 0; t_uchar md5[16]; t_uchar md5x[33]; sha1_context_t sha1c = 0; t_uchar sha1[20]; t_uchar sha1x[41]; int fd; md5c = make_md5_context (0); sha1c = make_sha1_context (0); fd = safe_open (data_file, O_RDONLY, 0); while (1) { t_uchar buf[4096]; ssize_t amt_read; amt_read = safe_read_retry (fd, buf, sizeof (buf)); if (!amt_read) break; else { md5_scan (md5c, buf, (size_t)amt_read); sha1_scan (sha1c, buf, (size_t)amt_read); } } md5_final (md5, md5c); sha1_final (sha1, sha1c); safe_close (fd); md5_ascii (md5x, md5); md5x[32] = 0; sha1_ascii (sha1x, sha1); sha1x[40] = 0; safe_printfmt (arch->txn_signature_fd, "md5 %s %s\n", description, md5x); rel_add_records (&arch->deferred_sha1_sums, rel_make_record (description, sha1x, 0), 0); free_md5_context (0, md5c); free_sha1_context (0, sha1c); } /* notify that an archive dir has changed */ void dir_changed (struct arch_pfs_archive * arch, t_uchar const *dir) { if (!arch->arch.http_blows) return; arch_pfs_update_listing_file (arch->pfs, (t_uchar *)dir); } int pfs_set_mirror (t_uchar ** errstr, struct arch_archive * archive, int enabled) { struct arch_pfs_archive *arch = (struct arch_pfs_archive *)archive; if (enabled) { t_uchar * metainfo_path = arch_fs_archive_meta_info_path (0); int result = arch_pfs_set_file_contents_in_dir (arch->pfs, metainfo_path, "mirror", archive->official_name, 1); if (result) { *errstr = str_save (0, "Failed to upload mirror file"); } lim_free (0, metainfo_path); return result; } else { t_uchar * metainfo_item_path = arch_fs_archive_meta_info_item_path (0, "mirror"); int result = arch_pfs_rm (arch->pfs, metainfo_item_path, 1); if (result) { *errstr = str_save (0, "Failed to delete mirror file"); } lim_free (0, metainfo_item_path); return result; } } void pfs_clear_cached (struct arch_pfs_archive * arch) { rel_free_table (arch->root_dirs); arch->root_dirs = NULL; } /* tag: Tom Lord Tue May 20 13:35:38 2003 (archive-pfs.c) */