/* archive-mirror.c: * **************************************************************** * Copyright (C) 2003 Tom Lord * * See the file "COPYING" for further information about * the copyright and warranty status of this work. */ #include "hackerlab/os/errno.h" #include "hackerlab/os/errno-to-string.h" #include "hackerlab/char/str.h" #include "hackerlab/fs/file-names.h" #include "hackerlab/vu/safe.h" #include "libfsutils/ensure-dir.h" #include "libfsutils/tmp-files.h" #include "libawk/relational.h" #include "libawk/associative.h" #include "libarch/my.h" #include "libarch/chatter.h" #include "libarch/archive.h" #include "libarch/namespace.h" #include "libarch/archive-mirror.h" #include "libarch/archive-setup.h" #include "libarch/patch-logs.h" /* __STDC__ prototypes for static functions */ static void mirror_revision (struct arch_archive * arch, struct arch_archive * to_arch, t_uchar * revision, t_uchar * prev_level, struct arch_archive_mirror_options * archive_mirror_opts); static int list_contains (rel_table list, t_uchar * elt); static void mirror_cached_revision(int is_cached, struct arch_archive_mirror_options * archive_mirror_opts, enum arch_revision_type type, struct arch_archive * arch, struct arch_archive * to_arch, t_uchar * revision, t_uchar * continuation_archive); static void mirror_revision_ancestry (int has_ancestry, struct arch_archive_mirror_options * archive_mirror_opts, enum arch_revision_type type, struct arch_archive * arch, struct arch_archive * to_arch, t_uchar * revision, t_uchar * continuation_archive); void arch_archive_mirror (int chatter_fd, struct arch_archive * arch, struct arch_archive * to_arch, t_uchar * limit, struct arch_archive_mirror_options * archive_mirror_opts) { t_uchar * only_category = 0; t_uchar * only_branch = 0; t_uchar * only_version = 0; t_uchar * only_revision = 0; rel_table categories = 0; rel_table to_categories = 0; rel_table missing_categories = 0; int branchless_version = 0; int c; if (!limit) arch_chatter (chatter_fd, "* mirroring %s from %s to %s\n", arch->official_name, arch->location, to_arch->location); else arch_chatter (chatter_fd, "* mirroring %s from %s to %s with limit %s\n", arch->official_name, arch->location, to_arch->location, limit); if (limit) { only_category = arch_parse_package_name (arch_ret_category, 0, limit); if (arch_valid_package_name (limit, arch_no_archive, arch_req_package, 1)) { only_branch = arch_parse_package_name (arch_ret_package, 0, limit); if (!str_cmp (only_branch, only_category)) { lim_free (0, only_branch); only_branch = 0; } else if (arch_valid_package_name (limit, arch_no_archive, arch_req_version, 1)) { only_version = arch_parse_package_name (arch_ret_package_version, 0, limit); if (arch_valid_package_name (limit, arch_no_archive, arch_req_patch_level, 1)) { only_revision = arch_parse_package_name (arch_ret_non_archive, 0, limit); } } } } if (only_version && ! only_branch) branchless_version = 1; categories = arch_archive_categories (arch); to_categories = arch_archive_categories (to_arch); missing_categories = rel_join (1, rel_join_output (1,0, -1), 0, 0, categories, to_categories); for (c = 0; c < rel_n_records (categories); ++c) { rel_table branches = 0; rel_table to_branches = 0; rel_table missing_branches = 0; int b; if (only_category && str_cmp (only_category, categories[c][0])) continue; branches = arch_archive_branches (arch, categories[c][0]); to_branches = arch_archive_branches (to_arch, categories[c][0]); missing_branches = rel_join (1, rel_join_output (1,0, -1), 0, 0, branches, to_branches); for (b = 0; b < rel_n_records (branches); ++b) { rel_table versions = 0; rel_table to_versions = 0; rel_table missing_versions = 0; int v; if (only_branch && str_cmp (only_branch, branches[b][0])) continue; if ( branchless_version && str_length(branches[b][0]) > 0) continue; versions = arch_archive_versions (arch, branches[b][0]); to_versions = arch_archive_versions (to_arch, branches[b][0]); missing_versions = rel_join (1, rel_join_output (1,0, -1), 0, 0, versions, to_versions); for (v = 0; v < rel_n_records (versions); ++v) { rel_table revisions = 0; rel_table to_revisions = 0; rel_table missing_revisions = 0; int r; if (only_version && str_cmp (only_version, versions[v][0])) continue; if (list_contains (missing_versions, versions[v][0])) { arch_setup_archive_simple_ext (chatter_fd, to_arch, versions[v][0]); } revisions = arch_archive_revisions (arch, versions[v][0], 2); to_revisions = arch_archive_revisions (to_arch, versions[v][0], 2); missing_revisions = rel_join (1, rel_join_output (1,0, -1), 0, 0, revisions, to_revisions); for (r = 0; r < rel_n_records (revisions); ++r) { if (only_revision && str_cmp (only_revision, revisions[r][0])) continue; if (list_contains (missing_revisions, revisions[r][0])) { t_uchar * prev_level = 0; if (r) { prev_level = arch_parse_package_name (arch_ret_patch_level, 0, revisions[r - 1][0]); } arch_chatter (chatter_fd, "** adding revision %s\n", revisions[r][0]); mirror_revision (arch, to_arch, revisions[r][0], prev_level, archive_mirror_opts); lim_free (0, prev_level); } else if (only_revision && 0 == str_cmp (only_revision, revisions[r][0])) { t_uchar * prev_level = 0; enum arch_revision_type type; int is_cached; int has_ancestry; arch_patch_id * continuation = NULL; arch_patch_id * revision = arch_patch_id_new_archive (arch->official_name, revisions[r][0]); if (r) { prev_level = arch_parse_package_name (arch_ret_patch_level, 0, revisions[r - 1][0]); } arch_chatter (chatter_fd, "** adding revision %s\n", revisions[r][0]); arch_revision_type (&type, &is_cached, &has_ancestry, arch, revision); if (type == arch_continuation_revision) { continuation = arch_get_continuation (arch, revision); } mirror_cached_revision (is_cached, archive_mirror_opts, type, arch, to_arch, revisions[r][0], continuation ? arch_patch_id_archive (continuation) : NULL); mirror_revision_ancestry (has_ancestry, archive_mirror_opts, type, arch, to_arch, revisions[r][0], continuation ? arch_patch_id_archive (continuation) : NULL); talloc_free (continuation); lim_free (0, prev_level); talloc_free (revision); } } rel_free_table (revisions); rel_free_table (to_revisions); rel_free_table (missing_revisions); } rel_free_table (versions); rel_free_table (to_versions); rel_free_table (missing_versions); } rel_free_table (branches); rel_free_table (to_branches); rel_free_table (missing_branches); } lim_free (0, only_category); lim_free (0, only_branch); lim_free (0, only_version); lim_free (0, only_revision); rel_free_table (categories); rel_free_table (to_categories); rel_free_table (missing_categories); } static void mirror_revision (struct arch_archive * arch, struct arch_archive * to_arch, t_uchar * revision, t_uchar * prev_level, struct arch_archive_mirror_options * archive_mirror_opts) { t_uchar * version = 0; t_uchar * this_level = 0; t_uchar * errstr = 0; enum arch_revision_type type; int is_cached; int has_ancestry; t_uchar * my_uid = 0; t_uchar * txn_id = 0; t_uchar * log = 0; t_uchar * continuation_archive = 0; int compatible_checksums = arch->signed_archive == to_arch->signed_archive; arch_patch_id * revision_patch = arch_patch_id_new_archive (arch->official_name, revision); version = arch_parse_package_name (arch_ret_package_version, 0, revision); this_level = arch_parse_package_name (arch_ret_patch_level, 0, revision); arch_revision_type (&type, &is_cached, &has_ancestry, arch, revision_patch); my_uid = arch_my_id_uid_default ("Unknown User "); txn_id = arch_generate_txn_id (); if (arch_archive_lock_revision (&errstr, to_arch, version, prev_level, my_uid, txn_id, this_level)) { safe_printfmt (2, "arch-mirror: unable to acquire revision lock (%s)\n url: %s\nrevision: %s/%s\n", errstr, to_arch->location, to_arch->official_name, revision); exit (2); } log = arch_archive_log (arch, revision); if (arch_archive_put_log (&errstr, to_arch, version, prev_level, my_uid, txn_id, log)) { safe_printfmt (2, "arch-mirror: unable to send log message to archive (%s)\n url: %s\nrevision: %s/%s\n", errstr, to_arch->location, to_arch->official_name, version); exit (2); } if (type == arch_import_revision) { t_uchar * tmp_path = 0; int fd; tmp_path = tmp_file_name_in_tmp (",,arch-mirror-changeset"); fd = safe_open (tmp_path, O_RDWR | O_CREAT | O_EXCL, 0000); safe_unlink (tmp_path); arch_get_import_targz (fd, arch, revision); safe_lseek (fd, (off_t)0, SEEK_SET); if (arch_archive_put_import_targz (&errstr, to_arch, version, prev_level, my_uid, txn_id, this_level, fd)) { safe_printfmt (2, "arch-mirror: unable to send import tree to archive (%s)\n url: %s\nrevision: %s/%s\n", errstr, to_arch->official_name, to_arch->location, version); exit (2); } safe_close (fd); lim_free (0, tmp_path); } else { t_uchar * tmp_path = 0; int fd; tmp_path = tmp_file_name_in_tmp (",,arch-mirror-full-text"); fd = safe_open (tmp_path, O_RDWR | O_CREAT | O_EXCL, 0000); safe_unlink (tmp_path); arch_get_patch_targz (fd, arch, revision); safe_lseek (fd, (off_t)0, SEEK_SET); if (arch_archive_put_changeset_targz (&errstr, to_arch, version, prev_level, my_uid, txn_id, this_level, fd)) { safe_printfmt (2, "arch-mirror: unable to send changeset to archive (%s)\n url: %s\nrevision: %s/%s\n", errstr, to_arch->official_name, to_arch->location, version); exit (2); } safe_close (fd); lim_free (0, tmp_path); } if (type == arch_continuation_revision) { arch_patch_id * continuation = NULL; arch_patch_id * revision_patch = arch_patch_id_new_archive (arch->official_name, revision); continuation = arch_get_continuation (arch, revision_patch); continuation_archive = str_save (0, arch_patch_id_archive (continuation)); if (arch_archive_put_continuation (&errstr, to_arch, version, prev_level, my_uid, txn_id, arch_patch_id_patch_id (continuation))) { safe_printfmt (2, "arch-mirror: unable to send tagged revision id to archive (%s)\n url: %s\nrevision: %s/%s\n", errstr, to_arch->official_name, to_arch->location, revision); exit (2); } talloc_free (continuation); talloc_free (revision_patch); } if (arch_mirror_revision_ready (&errstr, to_arch, compatible_checksums ? arch : NULL, version, prev_level, my_uid, txn_id, this_level)) { safe_printfmt (2, "arch-mirror: unable to complete phase-1 commit transaction (%s)\n url: %s\nrevision: %s/%s\n", errstr, to_arch->official_name, to_arch->location, version); exit (2); } if (arch_archive_finish_revision (&errstr, to_arch, version, prev_level, my_uid, txn_id, this_level)) { safe_printfmt (2, "arch-mirror: unable to complete commit transaction (%s)\n url: %s\nrevision: %s/%s\n", errstr, to_arch->official_name, to_arch->location, version); exit (2); } if (archive_mirror_opts->print_summary) { t_uchar * log = arch_archive_log (to_arch, revision); assoc_table headers = 0; t_uchar * body = 0; arch_parse_log (0, &headers, &body, log); arch_print_headers_summary (1, 4, headers, arch_include_summary); lim_free (0, log); free_assoc_table (headers); lim_free (0, body); } mirror_cached_revision(is_cached, archive_mirror_opts, type, arch, to_arch, revision, continuation_archive); mirror_revision_ancestry(has_ancestry, archive_mirror_opts, type, arch, to_arch, revision, continuation_archive); lim_free (0, version); lim_free (0, my_uid); lim_free (0, txn_id); lim_free (0, log); lim_free (0, continuation_archive); talloc_free (revision_patch); } void mirror_cached_revision(int is_cached, struct arch_archive_mirror_options * archive_mirror_opts, enum arch_revision_type type, struct arch_archive * arch, struct arch_archive * to_arch, t_uchar * revision, t_uchar * continuation_archive) { t_uchar * version = arch_parse_package_name (arch_ret_package_version, 0, revision); int compatible_checksums = arch->signed_archive == to_arch->signed_archive; if (is_cached && ((archive_mirror_opts->cachedrevs == arch_mirror_all_cachedrevs) || ((archive_mirror_opts->cachedrevs == arch_mirror_foreign_continuation_cachedrevs) && (type == arch_continuation_revision) && (str_cmp (arch->official_name, continuation_archive))))) { t_uchar * errstr = 0; t_uchar * tmp_path = 0; int fd; tmp_path = tmp_file_name_in_tmp (",,arch-mirror-archive-cached"); fd = safe_open (tmp_path, O_RDWR | O_CREAT | O_EXCL, 0000); safe_unlink (tmp_path); arch_get_cached_revision_targz (fd, arch, revision); safe_lseek (fd, (off_t)0, SEEK_SET); if (arch_archive_mirror_cached_targz (&errstr, to_arch, compatible_checksums ? arch : NULL, revision, fd)) { /* FIXME: mention the location? */ safe_printfmt (2, "arch-mirror: unable to send cached revision to archive (%s)\n revision: %s/%s\n", errstr, arch->official_name, version); exit (2); } safe_close (fd); lim_free (0, tmp_path); } else { t_uchar * errstr = 0; if (arch_archive_delete_cached (&errstr, to_arch, revision)) { /* FIXME: mention the location? */ safe_printfmt (2, "%s: unable to delete cached revision of %s (%s)\n", to_arch->official_name, revision, errstr); exit (1); } } lim_free (0, version); } void mirror_revision_ancestry(int has_ancestry, struct arch_archive_mirror_options * archive_mirror_opts, enum arch_revision_type type, struct arch_archive * arch, struct arch_archive * to_arch, t_uchar * revision, t_uchar * continuation_archive) { t_uchar * errstr = 0; t_uchar * tmp_path = 0; int fd; int compatible_checksums = arch->signed_archive == to_arch->signed_archive; /* remove the target ancestry if present */ arch_archive_delete_ancestry (NULL, to_arch, revision); /* if there is no ancestry, do nothing */ if (!has_ancestry) return; tmp_path = tmp_file_name_in_tmp (",,arch-mirror-archive-ancestry"); fd = safe_open (tmp_path, O_RDWR | O_CREAT | O_EXCL, 0000); /* FIXME: this isn't portable - windows cannot unlink in use files */ safe_unlink (tmp_path); arch_archive_get_ancestry (fd, arch, revision); safe_lseek (fd, (off_t)0, SEEK_SET); if (arch_archive_mirror_ancestry (&errstr, to_arch, compatible_checksums ? arch : NULL, revision, fd)) { /* FIXME: mention the location? */ safe_printfmt (2, "arch-mirror: unable to send cached revision to archive (%s)\n revision: %s/%s\n", errstr, arch->official_name, revision); exit (2); } safe_close (fd); lim_free (0, tmp_path); } static int list_contains (rel_table list, t_uchar * elt) { int x; for (x = 0; x < rel_n_records (list); ++x) { if (!str_cmp (elt, list[x][0])) return 1; } return 0; } /* tag: Tom Lord Mon Jun 9 23:29:24 2003 (archive-mirror.c) */