diff -r 17ce0bc36a49 -r 87cb4f9014f1 Tools/Rsync/Makefile.in --- a/Tools/Rsync/Makefile.in Tue Feb 23 21:03:38 2010 +0000 +++ b/Tools/Rsync/Makefile.in Fri Oct 29 12:32:07 2010 +1100 @@ -35,7 +35,8 @@ OBJS1=flist.o rsync.o generator.o receiver.o cleanup.o sender.o exclude.o \ util.o main.o checksum.o match.o syscall.o log.o backup.o OBJS2=options.o io.o compat.o hlink.o token.o uidlist.o socket.o hashtable.o \ - fileio.o batch.o clientname.o chmod.o acls.o xattrs.o + fileio.o batch.o clientname.o chmod.o acls.o xattrs.o fscache.o ntstreams.o \ + obfuscation.o OBJS3=progress.o pipe.o DAEMON_OBJ = params.o loadparm.o clientserver.o access.o connection.o authenticate.o popt_OBJS=popt/findme.o popt/popt.o popt/poptconfig.o \ diff -r 17ce0bc36a49 -r 87cb4f9014f1 Tools/Rsync/backup.c --- a/Tools/Rsync/backup.c Tue Feb 23 21:03:38 2010 +0000 +++ b/Tools/Rsync/backup.c Fri Oct 29 12:32:07 2010 +1100 @@ -143,7 +143,7 @@ #ifdef SUPPORT_XATTRS sx.xattr = NULL; #endif - if (!(file = make_file(rel, NULL, NULL, 0, NO_FILTERS))) + if (!(file = make_file(rel, NULL, NULL, 0, NO_FILTERS, rel))) continue; #ifdef SUPPORT_ACLS if (preserve_acls && !S_ISLNK(file->mode)) { @@ -224,7 +224,7 @@ sx.xattr = NULL; #endif - if (!(file = make_file(fname, NULL, NULL, 0, NO_FILTERS))) + if (!(file = make_file(fname, NULL, NULL, 0, NO_FILTERS, fname))) return 1; /* the file could have disappeared */ if (!(buf = get_backup_name(fname))) { diff -r 17ce0bc36a49 -r 87cb4f9014f1 Tools/Rsync/cleanup.c --- a/Tools/Rsync/cleanup.c Tue Feb 23 21:03:38 2010 +0000 +++ b/Tools/Rsync/cleanup.c Fri Oct 29 12:32:07 2010 +1100 @@ -191,6 +191,14 @@ /* FALLTHROUGH */ #include "case_N.h" + fscache_save(); + + /* FALLTHROUGH */ +#include "case_N.h" + obf_save(); + + /* FALLTHROUGH */ +#include "case_N.h" if (verbose > 2) { rprintf(FINFO, diff -r 17ce0bc36a49 -r 87cb4f9014f1 Tools/Rsync/exclude.c --- a/Tools/Rsync/exclude.c Tue Feb 23 21:03:38 2010 +0000 +++ b/Tools/Rsync/exclude.c Fri Oct 29 12:32:07 2010 +1100 @@ -36,6 +36,7 @@ extern int sanitize_paths; extern int protocol_version; extern int module_id; +extern char *obfuscation_file; extern char curr_dir[]; extern unsigned int curr_dir_len; @@ -1167,6 +1168,7 @@ static void send_rules(int f_out, struct filter_list_struct *flp) { struct filter_struct *ent, *prev = NULL; + char obf_pattern_buf[MAXPATHLEN], *pattern; for (ent = flp->head; ent; ent = ent->next) { unsigned int len, plen, dlen; @@ -1204,7 +1206,17 @@ if (f == f_out) continue; } - p = get_rule_prefix(ent->match_flags, ent->pattern, 1, &plen); + + if (obfuscation_file) { + obfuscate_pattern(obf_pattern_buf, MAXPATHLEN, ent->pattern); + pattern = obf_pattern_buf; + if (verbose > 2) + rprintf(FINFO, "obfuscating pattern \"%s\" -> \"%s\"\n", ent->pattern, obf_pattern_buf); + } + else + pattern = ent->pattern; + + p = get_rule_prefix(ent->match_flags, pattern, 1, &plen); if (!p) { rprintf(FERROR, "filter rules are too modern for remote rsync.\n"); @@ -1212,14 +1224,14 @@ } if (f_out < 0) continue; - len = strlen(ent->pattern); + len = strlen(pattern); dlen = ent->match_flags & MATCHFLG_DIRECTORY ? 1 : 0; if (!(plen + len + dlen)) continue; write_int(f_out, plen + len + dlen); if (plen) write_buf(f_out, p, plen); - write_buf(f_out, ent->pattern, len); + write_buf(f_out, pattern, len); if (dlen) write_byte(f_out, '/'); } diff -r 17ce0bc36a49 -r 87cb4f9014f1 Tools/Rsync/flist.c --- a/Tools/Rsync/flist.c Tue Feb 23 21:03:38 2010 +0000 +++ b/Tools/Rsync/flist.c Fri Oct 29 12:32:07 2010 +1100 @@ -72,6 +72,7 @@ extern int unsort_ndx; extern struct stats stats; extern char *filesfrom_host; +extern char *obfuscation_file; extern char curr_dir[MAXPATHLEN]; @@ -80,6 +81,8 @@ extern struct filter_list_struct filter_list; extern struct filter_list_struct daemon_filter_list; +extern int backup_nt_streams; + #ifdef ICONV_OPTION extern int filesfrom_convert; extern iconv_t ic_send, ic_recv; @@ -290,7 +293,7 @@ } static void send_directory(int f, struct file_list *flist, - char *fbuf, int len, int flags); + char *fbuf, int len, int flags, const char *parentdir); static const char *pathname, *orig_dir; static int pathname_len; @@ -630,6 +633,18 @@ stats.total_size += F_LENGTH(file); } +static void add_obf(struct file_struct *file) +{ + if (obfuscation_file) { + file->dirname_obf = file->dirname ? obf_lookup_dirname(file->dirname) : NULL; + file->basename_obf = obf_lookup_basename(file->basename); + } else { + file->dirname_obf = file->dirname; + file->basename_obf = file->basename; + } + +} + static struct file_struct *recv_file_entry(struct file_list *flist, int xflags, int f) { @@ -698,6 +713,9 @@ } #endif + if (obfuscation_file) + deobfuscate_fname(thisname, MAXPATHLEN, thisname); + if (*thisname) clean_fname(thisname, 0); @@ -1061,6 +1079,8 @@ if (S_ISREG(mode) || S_ISLNK(mode)) stats.total_size += file_length; + add_obf(file); + return file; } @@ -1075,12 +1095,13 @@ * "io_error |= IOERR_GENERAL" to avoid deletion of the file from the * destination if --delete is on. */ struct file_struct *make_file(const char *fname, struct file_list *flist, - STRUCT_STAT *stp, int flags, int filter_level) + STRUCT_STAT *stp, int flags, int filter_level, const char *real_fname) { static char *lastdir; static int lastdir_len = -1; struct file_struct *file; char thisname[MAXPATHLEN]; + char real_thisname[MAXPATHLEN]; char linkname[MAXPATHLEN]; int alloc_len, basename_len, linkname_len; int extra_len = file_extra_cnt * EXTRA_LEN; @@ -1098,15 +1119,24 @@ if (sanitize_paths) sanitize_path(thisname, thisname, "", 0, SP_DEFAULT); + if (strlcpy(real_thisname, real_fname, sizeof real_thisname) >= sizeof real_thisname) { + io_error |= IOERR_GENERAL; + rprintf(FERROR_XFER, "skipping overly long name: %s\n", real_fname); + return NULL; + } + clean_fname(real_thisname, 0); + if (sanitize_paths) + sanitize_path(real_thisname, real_thisname, "", 0, SP_DEFAULT); + if (stp && S_ISDIR(stp->st_mode)) { st = *stp; /* Needed for "symlink/." with --relative. */ *linkname = '\0'; /* make IBM code checker happy */ - } else if (readlink_stat(thisname, &st, linkname) != 0) { + } else if (readlink_stat(real_thisname, &st, linkname) != 0) { int save_errno = errno; /* See if file is excluded before reporting an error. */ if (filter_level != NO_FILTERS - && (is_excluded(thisname, 0, filter_level) - || is_excluded(thisname, 1, filter_level))) { + && (is_excluded(real_thisname, 0, filter_level) + || is_excluded(real_thisname, 1, filter_level))) { if (ignore_perishable && save_errno != ENOENT) non_perishable_cnt++; return NULL; @@ -1120,34 +1150,42 @@ * options was specified, so there's no need for the * extra lstat() if one of these options isn't on. */ if ((copy_links || copy_unsafe_links || copy_dirlinks) - && x_lstat(thisname, &st, NULL) == 0 + && x_lstat(real_thisname, &st, NULL) == 0 && S_ISLNK(st.st_mode)) { io_error |= IOERR_GENERAL; rprintf(FERROR_XFER, "symlink has no referent: %s\n", - full_fname(thisname)); + full_fname(real_thisname)); } else #endif { enum logcode c = am_daemon && protocol_version < 28 ? FERROR : FWARNING; io_error |= IOERR_VANISHED; - rprintf(c, "file has vanished: %s\n", - full_fname(thisname)); + rprintf(c, "make_file: file has vanished: %s\n", + full_fname(real_thisname)); } } else { io_error |= IOERR_GENERAL; rsyserr(FERROR_XFER, save_errno, "readlink_stat(%s) failed", - full_fname(thisname)); + full_fname(real_thisname)); } return NULL; } + /* NT streams files should not be directories, even if the base file is. */ + if (flags & FLAG_NT_STREAM) { + st.st_mode &= ~S_IFDIR; + st.st_mode |= S_IFREG; + } + + fscache_lookup(thisname, &st.st_size); + if (filter_level == NO_FILTERS) goto skip_filters; if (S_ISDIR(st.st_mode)) { if (!xfer_dirs) { - rprintf(FINFO, "skipping directory %s\n", thisname); + rprintf(FINFO, "skipping directory %s\n", real_thisname); return NULL; } /* -x only affects dirs because we need to avoid recursing @@ -1159,7 +1197,7 @@ if (verbose > 1) { rprintf(FINFO, "[%s] skipping mount-point dir %s\n", - who_am_i(), thisname); + who_am_i(), real_thisname); } return NULL; } @@ -1169,7 +1207,7 @@ } else flags &= ~FLAG_CONTENT_DIR; - if (is_excluded(thisname, S_ISDIR(st.st_mode) != 0, filter_level)) { + if (is_excluded(real_thisname, S_ISDIR(st.st_mode) != 0, filter_level)) { if (ignore_perishable) non_perishable_cnt++; return NULL; @@ -1179,7 +1217,7 @@ #ifdef SUPPORT_LINKS if (!S_ISLNK(st.st_mode)) #endif - if (access(thisname, R_OK) != 0) + if (access(real_thisname, R_OK) != 0) return NULL; } @@ -1299,6 +1337,10 @@ if (always_checksum && am_sender && S_ISREG(st.st_mode)) file_checksum(thisname, tmp_sum, st.st_size); + // XXX if we use thisname here then --checksum option will + // result in nt streams files always being sent - dmo. + // However, at this stage the nt streams file doesn't exist so + // I'm not sure what else to do. if (am_sender) F_PATHNAME(file) = pathname; @@ -1314,6 +1356,8 @@ if (unsort_ndx) F_NDX(file) = dir_count; + add_obf(file); + return file; } @@ -1325,11 +1369,11 @@ static struct file_struct *send_file_name(int f, struct file_list *flist, const char *fname, STRUCT_STAT *stp, - int flags, int filter_level) + int flags, int filter_level, const char *real_fname) { struct file_struct *file; - file = make_file(fname, flist, stp, flags, filter_level); + file = make_file(fname, flist, stp, flags, filter_level, real_fname); if (!file) return NULL; @@ -1413,13 +1457,13 @@ #endif } else #endif - f_name(file, fbuf); + f_name_maybe_obf(file, fbuf); #ifdef SUPPORT_ACLS if (preserve_acls && !S_ISLNK(file->mode)) { sx.st.st_mode = file->mode; sx.acc_acl = sx.def_acl = NULL; - if (get_acl(fname, &sx) < 0) { + if (get_acl(real_fname, &sx) < 0) { io_error |= IOERR_GENERAL; return NULL; } @@ -1428,7 +1472,7 @@ #ifdef SUPPORT_XATTRS if (preserve_xattrs) { sx.xattr = NULL; - if (get_xattr(fname, &sx) < 0) { + if (get_xattr(real_fname, &sx) < 0) { io_error |= IOERR_GENERAL; return NULL; } @@ -1483,7 +1527,8 @@ return; } save_filters = push_local_filters(fbuf, len); - send_directory(f, flist, fbuf, len, flags); + flags |= file->flags & FLAG_NT_STREAM_DIR; + send_directory(f, flist, fbuf, len, flags, file->dirname); pop_local_filters(save_filters); fbuf[ol] = '\0'; if (is_dot_dir) @@ -1597,11 +1642,12 @@ DIR_NEXT_SIBLING(dp) = -1; } -static void interpret_stat_error(const char *fname, int is_dir) +static void interpret_stat_error(const char *func_name, const char *fname, int is_dir) { if (errno == ENOENT) { io_error |= IOERR_VANISHED; - rprintf(FWARNING, "%s has vanished: %s\n", + rprintf(FWARNING, "%s: %s has vanished: %s\n", + func_name, is_dir ? "directory" : "file", full_fname(fname)); } else { io_error |= IOERR_GENERAL; @@ -1616,7 +1662,7 @@ * might call this with f set to -2, which also indicates that local filter * rules should be ignored. */ static void send_directory(int f, struct file_list *flist, char *fbuf, int len, - int flags) + int flags, const char *parentdir) { struct dirent *di; unsigned remainder; @@ -1626,12 +1672,20 @@ int start = flist->used; int filter_level = f == -2 ? SERVER_FILTERS : ALL_FILTERS; + char dirname[MAXPATHLEN]; + strncpy(dirname, fbuf, len); + dirname[len] = '\0'; + assert(flist != NULL); - if (!(d = opendir(fbuf))) { + if (!(d = opendir(flags & FLAG_NT_STREAM_DIR ? (parentdir ? parentdir : ".") : fbuf))) { if (errno == ENOENT) { if (am_sender) /* Can abuse this for vanished error w/ENOENT: */ - interpret_stat_error(fbuf, True); + interpret_stat_error("send_directory", fbuf, True); + return; + } else if (errno == ENOTDIR) { + if (am_sender) + rprintf(FWARNING, "opendir: not a directory: %s\n", fbuf); return; } io_error |= IOERR_GENERAL; @@ -1645,7 +1699,10 @@ *p = '\0'; remainder = MAXPATHLEN - (p - fbuf); + if (flags & FLAG_NT_STREAM_DIR) + flags |= FLAG_NT_STREAM; for (errno = 0, di = readdir(d); di; errno = 0, di = readdir(d)) { + char orig_ntbuf[MAXPATHLEN]; char *dname = d_name(di); if (dname[0] == '.' && (dname[1] == '\0' || (dname[1] == '.' && dname[2] == '\0'))) @@ -1665,8 +1722,15 @@ continue; } - send_file_name(f, flist, fbuf, NULL, flags, filter_level); + ntstreams_orig_filename(orig_ntbuf, MAXPATHLEN, dirname, dname); + send_file_name(f, flist, fbuf, NULL, flags, filter_level, orig_ntbuf); } + if (backup_nt_streams && !(flags & FLAG_NT_STREAM_DIR)) { + char ntbuf[MAXPATHLEN]; + ntstreams_dirname(ntbuf, MAXPATHLEN, dirname); + send_file_name(f, flist, ntbuf, NULL, flags | FLAG_NT_STREAM_DIR, filter_level, dirname); + } + fbuf[len] = '\0'; @@ -1677,11 +1741,13 @@ closedir(d); - if (f >= 0 && recurse && !divert_dirs) { + if (f >= 0 && recurse) { int i, end = flist->used - 1; /* send_if_directory() bumps flist->used, so use "end". */ - for (i = start; i <= end; i++) - send_if_directory(f, flist, flist->files[i], fbuf, len, flags); + for (i = start; i <= end; i++) { + if (!divert_dirs) + send_if_directory(f, flist, flist->files[i], fbuf, len, flags); + } } } @@ -1739,14 +1805,14 @@ for (slash = start; (slash = strchr(slash+1, '/')) != NULL; ) { *slash = '\0'; - file = send_file_name(f, flist, fname, NULL, flags, ALL_FILTERS); + file = send_file_name(f, flist, fname, NULL, flags, ALL_FILTERS, fname); depth++; if (!inc_recurse && file && S_ISDIR(file->mode)) change_local_filter_dir(fname, strlen(fname), depth); *slash = '/'; } - file = send_file_name(f, flist, fname, NULL, flags, ALL_FILTERS); + file = send_file_name(f, flist, fname, NULL, flags, ALL_FILTERS, fname); if (inc_recurse) { if (file && !S_ISDIR(file->mode)) file = NULL; @@ -1814,12 +1880,12 @@ if (one_file_system) { STRUCT_STAT st; if (link_stat(fbuf, &st, copy_dirlinks) != 0) { - interpret_stat_error(fbuf, True); + interpret_stat_error("send1extra (1)", fbuf, True); return; } filesystem_dev = st.st_dev; } - send_directory(f, flist, fbuf, dlen, flags); + send_directory(f, flist, fbuf, dlen, flags | (file->flags & FLAG_NT_STREAM_DIR), file->dirname); } if (!relative_paths) @@ -1849,12 +1915,12 @@ if (name_type != NORMAL_NAME) { STRUCT_STAT st; if (link_stat(fbuf, &st, 1) != 0) { - interpret_stat_error(fbuf, True); + interpret_stat_error("send1extra (2)", fbuf, True); continue; } - send_file_name(f, flist, fbuf, &st, FLAG_TOP_DIR | flags, ALL_FILTERS); + send_file_name(f, flist, fbuf, &st, FLAG_TOP_DIR | flags, ALL_FILTERS, fbuf); } else - send_file_name(f, flist, fbuf, NULL, FLAG_TOP_DIR | flags, ALL_FILTERS); + send_file_name(f, flist, fbuf, NULL, FLAG_TOP_DIR | flags, ALL_FILTERS, fbuf); } free(relname_list); @@ -2100,7 +2166,7 @@ if (*fn == '.' && fn[1] == '/' && !implied_dot_dir) { send_file_name(f, flist, ".", NULL, (flags | FLAG_IMPLIED_DIR) & ~FLAG_CONTENT_DIR, - ALL_FILTERS); + ALL_FILTERS, "."); implied_dot_dir = 1; } len = clean_fname(fn, CFN_KEEP_TRAILING_SLASH @@ -2192,7 +2258,7 @@ struct file_struct *file; file = send_file_name(f, flist, fbuf, &st, FLAG_TOP_DIR | FLAG_CONTENT_DIR | flags, - NO_FILTERS); + NO_FILTERS, fbuf); if (!file) continue; if (inc_recurse) { @@ -2201,12 +2267,12 @@ send_dir_depth = 0; change_local_filter_dir(fbuf, len, send_dir_depth); } - send_directory(f, flist, fbuf, len, flags); + send_directory(f, flist, fbuf, len, flags, file->dirname); } } else send_if_directory(f, flist, file, fbuf, len, flags); } else - send_file_name(f, flist, fbuf, &st, flags, NO_FILTERS); + send_file_name(f, flist, fbuf, &st, flags, NO_FILTERS, fbuf); } gettimeofday(&end_tv, NULL); @@ -2876,6 +2942,7 @@ { int dif; const uchar *c1, *c2; + char *f1_dirname, *f2_dirname, *f1_basename, *f2_basename; enum fnc_state state1, state2; enum fnc_type type1, type2; enum fnc_type t_path = protocol_version >= 29 ? t_PATH : t_ITEM; @@ -2888,13 +2955,26 @@ if (!f2 || !F_IS_ACTIVE(f2)) return 1; - c1 = (uchar*)f1->dirname; - c2 = (uchar*)f2->dirname; + if (obfuscation_file) { + f1_dirname = (uchar*)f1->dirname_obf; + f2_dirname = (uchar*)f2->dirname_obf; + f1_basename = (uchar*)f1->basename_obf; + f2_basename = (uchar*)f2->basename_obf; + } else { + f1_dirname = (uchar*)f1->dirname; + f2_dirname = (uchar*)f2->dirname; + f1_basename = (uchar*)f1->basename; + f2_basename = (uchar*)f2->basename; + } + + + c1 = (uchar*)f1_dirname; + c2 = (uchar*)f2_dirname; if (c1 == c2) c1 = c2 = NULL; if (!c1) { type1 = S_ISDIR(f1->mode) ? t_path : t_ITEM; - c1 = (const uchar*)f1->basename; + c1 = (const uchar*)f1_basename; if (type1 == t_PATH && *c1 == '.' && !c1[1]) { type1 = t_ITEM; state1 = s_TRAILING; @@ -2907,7 +2987,7 @@ } if (!c2) { type2 = S_ISDIR(f2->mode) ? t_path : t_ITEM; - c2 = (const uchar*)f2->basename; + c2 = (const uchar*)f2_basename; if (type2 == t_PATH && *c2 == '.' && !c2[1]) { type2 = t_ITEM; state2 = s_TRAILING; @@ -2931,7 +3011,7 @@ break; case s_SLASH: type1 = S_ISDIR(f1->mode) ? t_path : t_ITEM; - c1 = (const uchar*)f1->basename; + c1 = (const uchar*)f1_basename; if (type1 == t_PATH && *c1 == '.' && !c1[1]) { type1 = t_ITEM; state1 = s_TRAILING; @@ -2961,7 +3041,7 @@ break; case s_SLASH: type2 = S_ISDIR(f2->mode) ? t_path : t_ITEM; - c2 = (const uchar*)f2->basename; + c2 = (const uchar*)f2_basename; if (type2 == t_PATH && *c2 == '.' && !c2[1]) { type2 = t_ITEM; state2 = s_TRAILING; @@ -3034,6 +3114,33 @@ return fbuf; } +char *f_name_obf(const struct file_struct *f, char *fbuf) +{ + if (!f || !F_IS_ACTIVE(f)) + return NULL; + + if (!fbuf) + fbuf = f_name_buf(); + + if (f->dirname_obf) { + int len = strlen(f->dirname_obf); + memcpy(fbuf, f->dirname_obf, len); + fbuf[len] = '/'; + strlcpy(fbuf + len + 1, f->basename_obf, MAXPATHLEN - (len + 1)); + } else + strlcpy(fbuf, f->basename_obf, MAXPATHLEN); + + return fbuf; +} + +char *f_name_maybe_obf(const struct file_struct *f, char *fbuf) +{ + if (obfuscation_file) + return f_name_obf(f, fbuf); + else + return f_name(f, fbuf); +} + /* Do a non-recursive scan of the named directory, possibly ignoring all * exclude rules except for the daemon's. If "dlen" is >=0, it is the length * of the dirname string, and also indicates that "dirname" is a MAXPATHLEN @@ -3058,7 +3165,7 @@ recurse = 0; xfer_dirs = 1; - send_directory(ignore_filter_rules ? -2 : -1, dirlist, dirname, dlen, FLAG_CONTENT_DIR); + send_directory(ignore_filter_rules ? -2 : -1, dirlist, dirname, dlen, FLAG_CONTENT_DIR, NULL); xfer_dirs = save_xfer_dirs; recurse = save_recurse; if (do_progress) diff -r 17ce0bc36a49 -r 87cb4f9014f1 Tools/Rsync/fscache.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Tools/Rsync/fscache.c Fri Oct 29 12:32:07 2010 +1100 @@ -0,0 +1,257 @@ +/* + * File size cache + * + * Copyright (C) 2010 Cortex IT + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, visit the http://fsf.org website. + */ + +#include "rsync.h" + +extern char *file_size_cache; +extern int verbose; + +struct fscache_node { + char *fname; + int64 size; + int age; + struct fscache_node *next; +}; + +/* Increment this if you change the file format */ +#define FSCACHE_VERSION 1 + +/* how long to keep old obftable entries that have not been needed in this run */ +#define FSCACHE_MAX_AGE 10 + +#define FSCACHE_TABLE_SIZE 65537 /* prime number */ +static struct fscache_node *fscache[FSCACHE_TABLE_SIZE]; + +void fscache_clear(void) +{ + unsigned i; + for (i = 0; i < FSCACHE_TABLE_SIZE; ++i) + { + struct fscache_node *p = fscache[i], *tmp; + while (p) + { + tmp = p; + p = p->next; + free(tmp->fname); + free(tmp); + } + fscache[i] = NULL; + } +} + +static unsigned hash(const char *fname) +{ + unsigned n = 0; + const char *p; + for (p = fname; *p; ++p) + n = (64 * n + *p) % FSCACHE_TABLE_SIZE; + return n; +} + +static char *fix_fname(const char *fname) +{ + char *ret = normalize_path((char *)fname, 1, NULL); +#ifdef __CYGWIN__ + /* This is a nasty hack to remove the drive letter from the start of the path. + * The reason we do this is because we want to be able to use this with VSS snapshots + * and the drive letter mapped to the VSS snapshot is not guaranteed to be the same from + * one run to the next. */ + if (strstr(ret, "/cygdrive/") == ret && strlen(ret) > 12) { + char *old = ret; + ret = strdup(ret + 12); + free(old); + } +#endif + return ret; +} + +int fscache_lookup(const char *fname, int64 *pSize) +{ + if (!file_size_cache) + return 0; + struct fscache_node *p; + char *norm_fname = fix_fname(fname); + + if (verbose > 3) + rprintf(FINFO, "fscache lookup %s\n", norm_fname); + + for (p = fscache[hash(norm_fname)]; p; p = p->next) + { + int cmp = strcmp(p->fname, norm_fname); + if (cmp == 0) { + *pSize = p->size; + p->age = 0; /* reset age */ + if (verbose > 2) + rprintf(FINFO, "fscache found %s, size %ld\n", norm_fname, (long)*pSize); + return 1; + } + if (cmp > 0) + break; + } + if (verbose > 2) + rprintf(FINFO, "fscache %s not found\n", norm_fname); + return 0; +} + +static struct fscache_node *new_node(char *fname, int64 size, int age, struct fscache_node *next) +{ + struct fscache_node *p = new(struct fscache_node); + p->fname = fname; + p->size = size; + p->age = age; + p->next = next; + return p; +} + +static void fscache_insert_internal(char *fname, int64 size, int age) +{ + struct fscache_node **p; + if (verbose > 3) + rprintf(FINFO, "fscache insert %s\n", fname); + + for (p = fscache + hash(fname); *p; p = &(*p)->next) + { + int cmp = strcmp((*p)->fname, fname); + if (cmp == 0) { + // Update existing node. + (*p)->size = size; + (*p)->age = age; + if (verbose > 2) + rprintf(FINFO, "fscache updating %s, size %ld, age %d\n", fname, (long)size, age); + return; + } + if (cmp > 0) + break; + } + *p = new_node(fname, size, age, *p); + if (verbose > 2) + rprintf(FINFO, "fscache adding %s, size %ld, age %d, addr %p, next %p\n", fname, (long)size, age, (void *)(*p), (void *)(*p)->next); +} + +void fscache_insert(const char *fname, int64 size) +{ + if (!file_size_cache) + return; + char *norm_fname = fix_fname(fname); + fscache_insert_internal(norm_fname, size, 0); +} + + +int fscache_load() +{ + int fd; + int32 version; + char fname_buf[MAXPATHLEN]; + int64 size; + + fscache_clear(); + + fd = do_open(file_size_cache, O_RDONLY, 0); + if (fd == -1) { + if (verbose > 0) + if (errno == ENOENT) + rprintf(FINFO, "fscache file %s: does not exist, a new file will be created\n", file_size_cache); + else + rprintf(FINFO, "fscache file %s: could not open for loading\n", file_size_cache); + return 0; + } + if (verbose > 0) + rprintf(FINFO, "fscache file %s: loading\n", file_size_cache); + + version = read_int(fd); + if (version > FSCACHE_VERSION) { + if (verbose > 0) + rprintf(FINFO, "fscache file %s: version %d > latest known version (%d), ignoring\n", file_size_cache, version, FSCACHE_VERSION); + return 0; + } + + while (1) { + int age = 0; + read_vstring(fd, fname_buf, MAXPATHLEN); + if (strlen(fname_buf) == 0) + break; // Empty string signifies that we've reached the end of the file. + + size = read_longint(fd); + + if (version >= 1) + age = read_int(fd); + age++; + fscache_insert_internal(strdup(fname_buf), size, age); + } + close(fd); + if (verbose > 0) + rprintf(FINFO, "fscache file %s: loaded\n", file_size_cache); + return 1; +} + +static void write_node(int fd, struct fscache_node *p) +{ + if (p->age > FSCACHE_MAX_AGE) { + if (verbose > 2) + rprintf(FINFO, "fscache discarding entry %s, size %ld, age %d\n", p->fname, (long)p->size, p->age); + return; + } + if (verbose > 2) + rprintf(FINFO, "fscache saving entry %s, size %ld, age %d\n", p->fname, (long)p->size, p->age); + write_vstring(fd, p->fname, strlen(p->fname)); + write_longint(fd, p->size); + write_int(fd, p->age); +} + +void fscache_save(void) +{ + int fd; + unsigned i; + char tmp[MAXPATHLEN]; + + if (!file_size_cache) + return; + + get_tmpname(tmp, file_size_cache); + + fd = do_mkstemp(tmp, S_IRUSR|S_IWUSR); + if (fd == -1) { + if (verbose > 0) + rprintf(FINFO, "fscache file %s: could not open for saving\n", tmp); + return; + } + if (verbose > 2) + rprintf(FINFO, "fscache file %s: saving\n", tmp); + + write_int(fd, (int32)FSCACHE_VERSION); + + for (i = 0; i < FSCACHE_TABLE_SIZE; ++i) { + struct fscache_node *p; + for (p = fscache[i]; p; p = p->next) + write_node(fd, p); + } + + // Write an empty string to mark the end of file. + write_vstring(fd, "", 0); + + close(fd); + + if (verbose > 2) + rprintf(FINFO, "fscache: renaming %s to %s\n", tmp, file_size_cache); + + if (robust_rename(tmp, file_size_cache, NULL, S_IRUSR|S_IWUSR) < 0) + rprintf(FERROR, "fscache: could not rename %s to %s\n", tmp, file_size_cache); + + return; +} diff -r 17ce0bc36a49 -r 87cb4f9014f1 Tools/Rsync/generator.c --- a/Tools/Rsync/generator.c Tue Feb 23 21:03:38 2010 +0000 +++ b/Tools/Rsync/generator.c Fri Oct 29 12:32:07 2010 +1100 @@ -62,6 +62,7 @@ extern int make_backups; extern int csum_length; extern int ignore_times; +extern int times_only; extern int size_only; extern OFF_T max_size; extern OFF_T min_size; @@ -75,6 +76,7 @@ extern int fuzzy_basis; extern int always_checksum; extern int checksum_len; +extern char *basis_filter; extern char *partial_dir; extern char *basis_dir[]; extern int compare_dest; @@ -98,6 +100,7 @@ extern int backup_suffix_len; extern struct file_list *cur_flist, *first_flist, *dir_flist; extern struct filter_list_struct daemon_filter_list; +extern int restore_nt_streams; int ignore_perishable = 0; int non_perishable_cnt = 0; @@ -114,6 +117,7 @@ static int need_retouch_dir_times; static int need_retouch_dir_perms; static const char *solo_file = NULL; +static char *filter_argv[MAX_FILTER_ARGS + 1] = { NULL }; /* For calling delete_item() and delete_dir_contents(). */ #define DEL_NO_UID_WRITE (1<<0) /* file/dir has our uid w/o write perm */ @@ -717,7 +721,7 @@ /* Perform our quick-check heuristic for determining if a file is unchanged. */ int unchanged_file(char *fn, struct file_struct *file, STRUCT_STAT *st) { - if (st->st_size != F_LENGTH(file)) + if (!times_only && st->st_size != F_LENGTH(file)) return 0; /* if always checksum is set then we use the checksum instead @@ -1275,6 +1279,7 @@ int is_dir = !S_ISDIR(file->mode) ? 0 : inc_recurse && ndx != cur_flist->ndx_start - 1 ? -1 : 1; + int is_ntstreams_file = 0; if (verbose > 2) rprintf(FINFO, "recv_generator(%s,%d)\n", fname, ndx); @@ -1365,7 +1370,27 @@ need_fuzzy_dirlist = 0; } - statret = link_stat(fname, &sx.st, keep_dirlinks && is_dir); + if (restore_nt_streams) + { + /* If the file is an nt streams file then it won't + * exist on the destination, however we don't want to + * restore it unless we're restoring the base + * (original) file. To accomplish this we stat the + * base file instead of the streams file so that + * unchanged_file() will use that instead. We then + * need to adjust the mode to make sure it is a regular + * file, not a directory. For this to work, + * --times-only option must also be in effect. */ + char ntstreams_fname[MAXPATHLEN]; + is_ntstreams_file = ntstreams_orig_filename(ntstreams_fname, MAXPATHLEN, file->dirname, file->basename); + statret = link_stat(ntstreams_fname, &sx.st, keep_dirlinks && is_dir); + if (is_ntstreams_file) { + sx.st.st_mode &= ~S_IFDIR; + sx.st.st_mode |= S_IFREG; + } + } else { + statret = link_stat(fname, &sx.st, keep_dirlinks && is_dir); + } stat_errno = errno; } @@ -1805,11 +1830,12 @@ do_unlink(partialptr); handle_partial_dir(partialptr, PDIR_DELETE); } - set_file_attrs(fname, file, &sx, NULL, maybe_ATTRS_REPORT); + if (!is_ntstreams_file) + set_file_attrs(fname, file, &sx, NULL, maybe_ATTRS_REPORT); if (itemizing) itemize(fnamecmp, file, ndx, statret, &sx, 0, 0, NULL); #ifdef SUPPORT_HARD_LINKS - if (preserve_hard_links && F_IS_HLINKED(file)) + if (preserve_hard_links && F_IS_HLINKED(file) && !is_ntstreams_file) finish_hard_link(file, fname, ndx, &sx.st, itemizing, code, -1); #endif if (remove_source_files != 1) @@ -1819,7 +1845,7 @@ send_msg_int(MSG_SUCCESS, ndx); goto cleanup; } - + if (append_mode > 0 && sx.st.st_size >= F_LENGTH(file)) { #ifdef SUPPORT_HARD_LINKS if (F_IS_HLINKED(file)) @@ -1843,7 +1869,7 @@ if (inplace && make_backups > 0 && fnamecmp_type == FNAMECMP_FNAME) { if (!(backupptr = get_backup_name(fname))) goto cleanup; - if (!(back_file = make_file(fname, NULL, NULL, 0, NO_FILTERS))) + if (!(back_file = make_file(fname, NULL, NULL, 0, NO_FILTERS, fname))) goto pretend_missing; if (copy_file(fname, backupptr, -1, back_file->mode, 1) < 0) { unmake_file(back_file); @@ -1859,6 +1885,11 @@ if (j >= 0) /* don't use changing file as future fuzzy basis */ fuzzy_dirlist->files[j]->flags |= FLAG_FILE_SENT; } + + if (is_ntstreams_file) { + statret = real_ret = -1; + goto notify_others; + } /* open the file */ if ((fd = do_open(fnamecmp, O_RDONLY, 0)) < 0) { @@ -1881,7 +1912,7 @@ close(fd); goto cleanup; } - if (!(back_file = make_file(fname, NULL, NULL, 0, NO_FILTERS))) { + if (!(back_file = make_file(fname, NULL, NULL, 0, NO_FILTERS, fname))) { close(fd); goto pretend_missing; } @@ -1912,6 +1943,57 @@ fnamecmp_type = FNAMECMP_BACKUP; } + if (fd != -1 && basis_filter) + { + char tmp[MAXPATHLEN]; + int fd_basis_filter = open_tmpfile(tmp, fnamecmp, file); + + /* make sure filter argv is set up */ + if (! filter_argv[0]) + tokenize_filter_args(filter_argv, basis_filter, "basis"); + + if (fd_basis_filter == -1) { + rprintf(FERROR, + "could not open temporary file for %s; bypassing basis filter %s\n", + fname, basis_filter); + } else { + int status; + pid_t pid; + char *filter_argv_subst[MAX_FILTER_ARGS + 1]; + enum filter_args_info fai = substitute_filter_args(filter_argv_subst, fnamecmp, tmp, filter_argv); + if (fai & fai_has_source) { + close(fd); + fd = -1; + } + if (fai & fai_has_dest) { + close(fd_basis_filter); + fd_basis_filter = -1; + } + pid = run_filter_on_file(filter_argv_subst, fd_basis_filter, fd); + if (fd >= 0) close(fd); + if (fd_basis_filter >= 0) close(fd_basis_filter); + wait_process_with_flush(pid, &status); + if (status != 0) { + rprintf(FERROR, + "bypassing basis filter %s; exited with code: %d\n", + basis_filter, status); + fd = do_open(fnamecmp, O_RDONLY, 0); + } else { + fd = do_open(tmp, O_RDONLY, 0); + fnamecmp = tmp; + if (do_fstat(fd,&sx.st) != 0) { + rsyserr(FERROR_XFER, errno, "fstat %s failed", + full_fname(fnamecmp)); + close(fd); + goto cleanup; + } + } + fnamecmp_type = FNAMECMP_FILTERED; + } + } + + + if (verbose > 3) { rprintf(FINFO, "gen mapped %s of size %.0f\n", fnamecmp, (double)sx.st.st_size); @@ -1936,10 +2018,14 @@ iflags |= ITEM_REPORT_CHANGE; if (fnamecmp_type != FNAMECMP_FNAME) iflags |= ITEM_BASIS_TYPE_FOLLOWS; - if (fnamecmp_type == FNAMECMP_FUZZY) + if (fnamecmp_type == FNAMECMP_FUZZY || fnamecmp_type == FNAMECMP_FILTERED) iflags |= ITEM_XNAME_FOLLOWS; itemize(fnamecmp, file, -1, real_ret, &real_sx, iflags, fnamecmp_type, - fuzzy_file ? fuzzy_file->basename : NULL); + fuzzy_file + ? fuzzy_file->basename + : fnamecmp_type == FNAMECMP_FILTERED + ? fnamecmp + : NULL); #ifdef SUPPORT_ACLS if (preserve_acls) free_acl(&real_sx); @@ -2220,6 +2306,9 @@ dflt_perms = (ACCESSPERMS & ~orig_umask); + if (basis_filter) + tokenize_filter_args(filter_argv, basis_filter, "basis"); + do { #ifdef SUPPORT_HARD_LINKS if (preserve_hard_links && inc_recurse) { diff -r 17ce0bc36a49 -r 87cb4f9014f1 Tools/Rsync/main.c --- a/Tools/Rsync/main.c Tue Feb 23 21:03:38 2010 +0000 +++ b/Tools/Rsync/main.c Fri Oct 29 12:32:07 2010 +1100 @@ -78,6 +78,8 @@ extern char curr_dir[MAXPATHLEN]; extern struct file_list *first_flist; extern struct filter_list_struct daemon_filter_list; +extern char *file_size_cache; +extern char *obfuscation_file; uid_t our_uid; int am_receiver = 0; /* Only set to 1 after the receiver/generator fork. */ @@ -137,7 +139,7 @@ } /* Wait for a process to exit, calling io_flush while waiting. */ -static void wait_process_with_flush(pid_t pid, int *exit_code_ptr) +void wait_process_with_flush(pid_t pid, int *exit_code_ptr) { pid_t waited_pid; int status; @@ -947,6 +949,22 @@ int child_main(int argc, char *argv[]) { + /* child_main acts as the server and receiver for a local transfer. + * We need to set obfuscation_file to NULL so that the receiver does not + * immediately de-obfuscate file names that have just been obfuscated by the + * sender. + * Note: + * The current implementation of file name obfuscation (via the + * --obfuscation-file option) assumes that the + * obfuscation/deobfuscation operation is done on the client and that + * whether we are obfuscating or deobfuscating depends on whether the + * client is the sender or receiver. This does not allow + * deobfuscation on local transfers where the "client" is always the + * sender. If we wanted to support that then we would need to add a + * new option to differentiate between obfuscation and deobfuscation + * operations. */ + obfuscation_file = NULL; + start_server(STDIN_FILENO, STDOUT_FILENO, argc, argv); return 0; } @@ -1126,7 +1144,13 @@ if (path) { /* source is remote */ char *dummy_host; int dummy_port = 0; - *argv = path; + if (obfuscation_file && strstr(path, OBF_ARGPATH_START_MARKER)) { + char tmp_buf[MAXPATHLEN]; + obfuscate_argpath(tmp_buf, MAXPATHLEN, path); + *argv = strdup(tmp_buf); + } else { + *argv = path; + } remote_argv = argv; remote_argc = argc; argv += argc - 1; @@ -1228,7 +1252,13 @@ rprintf(FERROR, "All source args must use the same port number.\n"); exit_cleanup(RERR_SYNTAX); } - remote_argv[i] = arg; + if (obfuscation_file && strstr(arg, OBF_ARGPATH_START_MARKER)) { + char tmp_buf[MAXPATHLEN]; + obfuscate_argpath(tmp_buf, MAXPATHLEN, arg); + remote_argv[i] = strdup(tmp_buf); + } else { + remote_argv[i] = arg; + } } } @@ -1447,6 +1477,12 @@ init_flist(); + if (file_size_cache) + fscache_load(); + + if (obfuscation_file) + obf_load(); + if ((write_batch || read_batch) && !am_server) { if (write_batch) write_batch_shell_file(orig_argc, orig_argv, argc); diff -r 17ce0bc36a49 -r 87cb4f9014f1 Tools/Rsync/ntstreams.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Tools/Rsync/ntstreams.c Fri Oct 29 12:32:07 2010 +1100 @@ -0,0 +1,528 @@ +/* + * NTFS backup/restore routines + * + * Copyright (C) 2010 Cortex IT + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, visit the http://fsf.org website. + */ + +#include "rsync.h" +#include +#include + +#define NTSTREAMS_DIR ".nt_streams" +#define BUFFER_SIZE (64*1024) +#define ERROR_BUFFER_SIZE 1024 + +extern int verbose; + +static char ntstreams_last_error[ERROR_BUFFER_SIZE]; +static char *ntstreams_last_error_what = ""; + +enum hopen_mode { hopen_read, hopen_write }; + +static HANDLE hopen(const char *path, enum hopen_mode mode) +{ + DWORD desiredAccess, shareMode, creationDisposition, flagsAndAttributes; + LPSECURITY_ATTRIBUTES securityAttributes = NULL; + HANDLE h, templateFile = NULL; + wchar_t *win_path = (wchar_t *)cygwin_create_path(CCP_POSIX_TO_WIN_W, (void *)path); + if (verbose > 3) + rprintf(FINFO, "cygwin_create_path(%s) = %ls\n", path, win_path); + switch (mode) { + case hopen_read: + desiredAccess = GENERIC_READ | ACCESS_SYSTEM_SECURITY; + shareMode = FILE_SHARE_READ; + creationDisposition = OPEN_EXISTING; + flagsAndAttributes = FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT; + break; + case hopen_write: + default: + /* desiredAccess = GENERIC_ALL | ACCESS_SYSTEM_SECURITY; + * GENERIC_ALL causes CreateFileW to give an "access denied" error. + * I don't know why, since this works in CortexIT.Replicator.BackupStreams. + */ + desiredAccess = WRITE_OWNER | WRITE_DAC | ACCESS_SYSTEM_SECURITY; + shareMode = FILE_SHARE_WRITE; + creationDisposition = OPEN_EXISTING; + flagsAndAttributes = FILE_FLAG_BACKUP_SEMANTICS; + break; + } + h = CreateFileW(win_path, desiredAccess, shareMode, securityAttributes, creationDisposition, flagsAndAttributes, templateFile); + free(win_path); + return h; +} + +// Size of the stream header without the stream name (which we treat seperately). +// #define SIZEOF_WIN32_STREAM_ID (sizeof(WIN32_STREAM_ID) - sizeof(WCHAR[ANYSIZE_ARRAY])) +#define SIZEOF_WIN32_STREAM_ID 20 + +/* Return values: + * 1: success + * 0: Posix error (error code in errno) + * -1: win32 error (use GetLastError to get error code)*/ +int ntfs_streams_to_fd(const char *path, int fd) +{ + int success = 1; + LPVOID context = NULL; + char buffer[BUFFER_SIZE]; + WIN32_STREAM_ID header; + DWORD bytesRead; + HANDLE h = hopen(path, hopen_read); + if (h == INVALID_HANDLE_VALUE) { + DWORD lastError = GetLastError(); + ntstreams_last_error_what = "CreateFile"; + FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, lastError, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + ntstreams_last_error, ERROR_BUFFER_SIZE, NULL); + return -1; + } + + while (1) { + LARGE_INTEGER bytesRemaining; + if (!BackupRead(h, (LPBYTE)(&header), SIZEOF_WIN32_STREAM_ID, &bytesRead, 0, 1, &context)) { + success = -1; + ntstreams_last_error_what = "BackupRead"; + break; + } + if (bytesRead == 0 || header.dwStreamId == 0) + break; /* end of file */ + + if (verbose > 3) + rprintf(FINFO, "BackupRead found header type %lx, size %lld, streamNameSize %ld, bytesRead %ld\n", header.dwStreamId, header.Size.QuadPart, header.dwStreamNameSize, bytesRead); + + bytesRemaining.QuadPart = SIZEOF_WIN32_STREAM_ID + header.Size.QuadPart + header.dwStreamNameSize - bytesRead; + if (header.dwStreamId == BACKUP_DATA) { + DWORD lowBytesSeeked, highBytesSeeked; + + /* Seek over the rest of the stream */ + if (!BackupSeek(h, bytesRemaining.LowPart, bytesRemaining.HighPart, &lowBytesSeeked, &highBytesSeeked, &context)) { + success = -1; + ntstreams_last_error_what = "BackupSeek"; + break; + } + } else { + if (write(fd, &header, SIZEOF_WIN32_STREAM_ID) < 0) { + success = 0; + ntstreams_last_error_what = "write"; + break; + } + while(bytesRemaining.QuadPart > 0) { + DWORD count = bytesRemaining.QuadPart > BUFFER_SIZE ? BUFFER_SIZE : bytesRemaining.LowPart; + if (!BackupRead(h, (LPBYTE)buffer, count, &bytesRead, 0, 1, &context)) { + success = -1; + ntstreams_last_error_what = "BackupRead"; + goto finish; + } + if (verbose > 3) + rprintf(FINFO, "BackupRead count %ld, bytesRead %ld\n", count, bytesRead); + if (bytesRead == 0) + break; + if (write(fd, buffer, bytesRead) < 0) { + success = 0; + ntstreams_last_error_what = "write"; + goto finish; + } + bytesRemaining.QuadPart -= bytesRead; + } + } + } +finish: + if (success == 0) { + /* Posix error */ + strncpy(ntstreams_last_error, strerror(errno), ERROR_BUFFER_SIZE); + } else if (success == -1) { + /* win32 error */ + DWORD lastError = GetLastError(); + FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, lastError, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + ntstreams_last_error, ERROR_BUFFER_SIZE, NULL); + } + BackupRead(h, NULL, 0, &bytesRead, 1, 1, &context); + CloseHandle(h); + return success; +} + +int fd_to_ntfs_streams(int fd, const char *path) +{ + int success = 1; + LPVOID context = NULL; + char buffer[BUFFER_SIZE]; + WIN32_STREAM_ID header; + ssize_t bytesRead; + DWORD bytesWritten; + HANDLE h = hopen(path, hopen_write); + if (h == INVALID_HANDLE_VALUE) { + DWORD lastError = GetLastError(); + ntstreams_last_error_what = "CreateFile"; + FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, lastError, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + ntstreams_last_error, ERROR_BUFFER_SIZE, NULL); + return -1; + } + + while (1) { + if ((bytesRead = read(fd, (void *)(&header), SIZEOF_WIN32_STREAM_ID)) < 0) { + success = 0; + ntstreams_last_error_what = "read"; + break; + } + if (bytesRead == 0 || header.dwStreamId == 0) + break; /* end of file */ + + if (header.dwStreamId == BACKUP_DATA) { + /* Seek over header name */ + if (header.dwStreamNameSize > 0) { + if (lseek(fd, header.dwStreamNameSize, SEEK_CUR) < 0) { + success = 0; + ntstreams_last_error_what = "lseek"; + break; + } + } + + /* Seek over the rest of the stream */ + if (lseek(fd, header.Size.QuadPart, SEEK_CUR) < 0) { + success = 0; + ntstreams_last_error_what = "lseek"; + break; + } + } else { + int64 bytesRemaining; + if (!BackupWrite(h, (LPBYTE)&header, SIZEOF_WIN32_STREAM_ID, &bytesWritten, 0, 1, &context)) { + success = -1; + ntstreams_last_error_what = "BackupWrite (header)"; + break; + } + bytesRemaining = header.Size.QuadPart + header.dwStreamNameSize; + while(bytesRemaining > 0) { + size_t count = bytesRemaining > BUFFER_SIZE ? BUFFER_SIZE : (size_t)bytesRemaining; + if ((bytesRead = read(fd, (void *)buffer, count)) < 0) { + success = 0; + ntstreams_last_error_what = "read"; + goto finish; + } + if (bytesRead == 0) + goto finish; /* end of file */ + if (!BackupWrite(h, (LPBYTE)buffer, (DWORD)count, &bytesWritten, 0, 1, &context)) { + success = -1; + ntstreams_last_error_what = "BackupWrite (data)"; + goto finish; + } + bytesRemaining -= count; + } + } + } +finish: + if (success == 0) { + /* Posix error */ + strncpy(ntstreams_last_error, strerror(errno), ERROR_BUFFER_SIZE); + } else if (success == -1) { + /* win32 error */ + DWORD lastError = GetLastError(); + FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, lastError, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + ntstreams_last_error, ERROR_BUFFER_SIZE, NULL); + } + BackupWrite(h, NULL, 0, &bytesWritten, 1, 1, &context); + CloseHandle(h); + return success; +} + +void ntstreams_dirname(char *dest, size_t destsize, const char *dirname) +{ + if (!dirname || strcmp(dirname, ".") == 0) + strncpy(dest, NTSTREAMS_DIR, destsize); + else + pathjoin(dest, destsize, dirname, NTSTREAMS_DIR); +} + +size_t ntstreams_filename(char *dest, char *orig, size_t destsize, const char *dirname, const char *basename) +{ + char tmp[MAXPATHLEN]; + if (!dirname) + dirname = "."; + ntstreams_dirname(tmp, MAXPATHLEN, dirname); + if (strcmp(dirname, ".") == 0) + strncpy(orig, basename, destsize); + else + pathjoin(orig, destsize, dirname, basename); + return pathjoin(dest, destsize, tmp, basename); +} + +int ntstreams_orig_filename(char *dest, size_t destsize, const char *dirname, const char *basename) +{ + char *p, tmp[MAXPATHLEN]; + int ret = 0; + strncpy(tmp, dirname ? dirname : ".", MAXPATHLEN); + p = strstr(tmp, NTSTREAMS_DIR); + if (p) { + *p = '\0'; + ret = 1; + } + if (strlen(tmp) == 0 || strcmp(tmp, ".") == 0) + strncpy(dest, basename, destsize); + else + pathjoin(dest, destsize, tmp, basename); + return ret; +} + +char *ntstreams_get_last_error_message() +{ + return ntstreams_last_error; +} + +const char *ntstreams_get_last_error_what() +{ + return ntstreams_last_error_what; +} + +int set_privilege_backup(int bEnablePrivilege) +{ + return set_privilege(SE_BACKUP_NAME, bEnablePrivilege); +} + +int set_privilege_restore(int bEnablePrivilege) +{ + return set_privilege(SE_RESTORE_NAME, bEnablePrivilege); +} + +int set_privilege_security(int bEnablePrivilege) +{ + return set_privilege(SE_SECURITY_NAME, bEnablePrivilege); +} + +int set_privilege_take_ownership(int bEnablePrivilege) +{ + return set_privilege(SE_TAKE_OWNERSHIP_NAME, bEnablePrivilege); +} + +int set_privilege(const char *lpszPrivilege, int bEnablePrivilege) +{ + HANDLE hToken; + TOKEN_PRIVILEGES tp; + LUID luid; + + if (!OpenProcessToken(GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES, &hToken)) + goto handle_error; + + if (!LookupPrivilegeValue(NULL, lpszPrivilege, &luid)) + goto handle_error; + + tp.PrivilegeCount = 1; + tp.Privileges[0].Luid = luid; + if (bEnablePrivilege) + tp.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED; + else + tp.Privileges[0].Attributes = 0; + + if (!AdjustTokenPrivileges(hToken, FALSE, &tp, sizeof(TOKEN_PRIVILEGES), (PTOKEN_PRIVILEGES) NULL, (PDWORD) NULL)) + goto handle_error; + + if (GetLastError() == ERROR_NOT_ALL_ASSIGNED) + goto handle_error; + return TRUE; + +handle_error: + FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, GetLastError(), MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + ntstreams_last_error, ERROR_BUFFER_SIZE, NULL); + return FALSE; +} + +struct restore_list { + char *basename; + time_t modtime; + mode_t mode; + struct restore_list *next; +}; + +static void cleanup_restore_list(struct restore_list *list) +{ + while (list) { + struct restore_list *p = list; + list = list->next; + free(p->basename); + free(p); + } +} + +struct restore_stack_node { + char *dirname; + struct restore_list *restore_list; + struct restore_stack_node *next; +}; + +struct restore_stack { + struct restore_stack_node* top; +}; + +struct restore_stack restore_stack; + +static void push(struct restore_stack *stack, const char *dirname) +{ + struct restore_stack_node *node = new(struct restore_stack_node); + if (verbose > 3) + rprintf(FINFO, "ntstreams push %s\n", dirname); + node->dirname = strdup(dirname); + node->restore_list = NULL; + node->next = stack->top; + stack->top = node; +} + +static void cleanup_restore_stack_node(struct restore_stack_node *node) +{ + if (!node) + return; + free(node->dirname); + cleanup_restore_list(node->restore_list); + free(node); +} + +static void pop(struct restore_stack *stack) +{ + if (stack->top) { + struct restore_stack_node *p = stack->top; + if (verbose > 3) + rprintf(FINFO, "ntstreams pop %s\n", stack->top->dirname); + stack->top = p->next; + cleanup_restore_stack_node(p); + } +} + +static void add_fname(struct restore_stack *stack, const struct file_struct *file) +{ + if (!stack->top) + return; + struct restore_list *list_node = new(struct restore_list); + list_node->basename = strdup(file->basename); + list_node->modtime = file->modtime; + list_node->mode = file->mode; + list_node->next = stack->top->restore_list; + stack->top->restore_list = list_node; +} + +static void restore_streams_and_remove(const char *dirname, const char *basename, time_t modtime, mode_t mode) +{ + char streams_fname[MAXPATHLEN], orig_fname[MAXPATHLEN]; + int fd; + ntstreams_filename(streams_fname, orig_fname, MAXPATHLEN, dirname, basename); + + /* Can't restore streams for "." without knowing its real name. */ + if (strcmp(orig_fname, ".") == 0) + return; + + fd = do_open(streams_fname, O_RDONLY, 0); + if (fd == -1) { + /* If the file was unchanged then there won't be a corresponding streams file. + * Therefore we need to ignore any errors with opening the streams file. + * rsyserr(FERROR, errno, "restore_streams_and_remove: open failed for %s", full_fname(streams_fname)); + */ + return; + } + if (verbose > 2) { + rprintf(FINFO, "restoring NT streams for %s\n", orig_fname); + } + if (fd_to_ntfs_streams(fd, orig_fname) <= 0) { + rprintf(FERROR, "fd_to_ntfs_streams %s failed on %s, skipping nt streams: %s\n", + ntstreams_get_last_error_what(), orig_fname, ntstreams_get_last_error_message()); + close(fd); + return; + } + close(fd); + if (unlink(streams_fname) == -1) { + rsyserr(FERROR, errno, "restore_streams_and_remove: unlink failed for %s", full_fname(streams_fname)); + } + if (set_modtime(orig_fname, modtime, mode) != 0) + { + rsyserr(FERROR, errno, "restore_streams_and_remove: set_modtime failed for %s", full_fname(orig_fname)); + } +} + +static void remove_ntstreams_dir(const char *dirname) +{ + char streams_dir[MAXPATHLEN]; + ntstreams_dirname(streams_dir, MAXPATHLEN, dirname); + if (rmdir(streams_dir) == -1) { + /* Don't report the error because the directory may not exist anyway. + * rsyserr(FERROR, errno, "remove_ntstreams_dir: rmdir failed for %s", full_fname(streams_dir)); + */ + } +} + +static int is_subdir_or_eq(const char *p1, const char *p2) +{ + int len = strlen(p2); + return strstr(p1, p2) == p1 && (p1[len] == '\0' || p1[len] == '/'); +} + +static int is_ntstreams_dir_or_file(const char *s) +{ + char *p = strstr(s, NTSTREAMS_DIR); + if (!p) + return 0; + p += strlen(NTSTREAMS_DIR); + return *p == '/' || *p == '\0'; +} + +void ntstreams_restore_update(const char *fname, const struct file_struct *file) +{ + static const char *dotdir = "."; + struct restore_stack *stack = &restore_stack; + if (fname && is_ntstreams_dir_or_file(fname)) + return; + + if (!stack->top) + push(stack, dotdir); + + const char *dirname = file && file->dirname ? file->dirname : dotdir; + while (stack->top && (!fname || (strcmp(stack->top->dirname, dotdir) != 0 && !is_subdir_or_eq(dirname, stack->top->dirname)))) + { + struct restore_list *rl; + for(rl = stack->top->restore_list; rl; rl = rl->next) { + restore_streams_and_remove(stack->top->dirname, rl->basename, rl->modtime, rl->mode); + } + remove_ntstreams_dir(stack->top->dirname); + pop(stack); + } + + // The stack is empty so we're done. + if (!stack->top) + return; + + if (dirname != dotdir) { + // Top of stack is now a parent directory (or equal to) dirname. + // Now push each component of dirname onto the stack. + char buf[MAXPATHLEN], *b = buf; + if (strcmp(stack->top->dirname, dotdir) != 0) { + strncpy(buf, stack->top->dirname, MAXPATHLEN); + b += strlen(b); + *b++ = '/'; + } + const char *p = dirname + (strcmp(stack->top->dirname, dotdir) == 0 ? 0 : strlen(stack->top->dirname)); + if (*p == '/') + p++; + while (*p) { + do { + *b++ = *p++; + } + while (*p && *p != '/'); + *b = '\0'; + push(stack, buf); + } + } + + if (!fname) + return; + add_fname(stack, file); +} diff -r 17ce0bc36a49 -r 87cb4f9014f1 Tools/Rsync/obfuscation.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Tools/Rsync/obfuscation.c Fri Oct 29 12:32:07 2010 +1100 @@ -0,0 +1,463 @@ +/* + * File name obfuscation + * + * Copyright (C) 2010 Cortex IT + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, visit the http://fsf.org website. + */ + +/* WARNING: + * This module assumes that the local file system is case-insensitive (e.g. NTFS or FAT). + * If you want to use it on a case-sensitive file system then you should change + * strcasecmp calls to strcmp. + */ + +#include "rsync.h" + +/* Increment this if you change the file format */ +#define OBF_VERSION 1 + +#define OBF_KEY_LENGTH 8 +#define OBF_CHARS_LENGTH 37 + +#define OBF_TABLE_SIZE 65537 /* prime number */ + +/* how long to keep obftable entries that have not been needed in this run */ +#define OBF_MAX_AGE 10 + +extern char *obfuscation_file; +extern int verbose; +extern char *partial_dir; + +struct obf_node { + const char *key; + const char *value; + int age; + struct obf_node *next; +}; + +static struct obf_node *obf_table[OBF_TABLE_SIZE], *deobf_obf_table[OBF_TABLE_SIZE], *obf_dirname_table[OBF_TABLE_SIZE]; + +static const char obf_chars[] = "_0123456789abcdefghijklmnopqrstuvwxyz"; + +static int obf_table_changed = 0; + +/* return a random character from obf_chars. + */ +static char obf(void) +{ + static int n = 0; + static int seeded = 0; + int m; + + /* If this is the first call to obf() then + * seed the random number generator. + */ + if (!seeded) { + srand(time(NULL)); + seeded = 1; + } + + if (n == 0) + n = rand(); + m = n % OBF_CHARS_LENGTH; + n /= OBF_CHARS_LENGTH; + return obf_chars[m]; +} + +static void obfuscate(char *value) +{ + int i; + for (i = 0; i < OBF_KEY_LENGTH; ++i) + value[i] = obf(); + value[OBF_KEY_LENGTH] = '\0'; +} + +static unsigned hash(const char *basename) +{ + unsigned n = 0; + const char *p; + for (p = basename; *p; ++p) + n = (64 * n + tolower((int)*p)) % OBF_TABLE_SIZE; + /* tolower() ensures that hashing is case-insensitive. */ + return n; +} + +/* Lookup is case insensitive. NOTE: this will cause problems if deobfuscating to a file system where case is significant. */ +static const char *lookup_key(struct obf_node **table, const char *key) +{ + struct obf_node *p; + for (p = table[hash(key)]; p; p = p->next) { + int cmp = strcasecmp(p->key, key); + if (cmp == 0) { + p->age = 0; /* reset age */ + return p->value; + } + if (cmp > 0) + return NULL; + } + return NULL; +} + +static int contains_key(struct obf_node **table, const char *key) +{ + return lookup_key(table, key) != NULL; +} + +static struct obf_node *new_node(const char *key, const char *value, int age, struct obf_node *next) +{ + struct obf_node *p = new(struct obf_node); + p->key = key; + p->value = value; + p->age = age; + p->next = next; + return p; +} + +/* Insert the key if it's not there. + * Case is preserved, but comparison with existing keys is case-insensitive */ +static void insert_key_value(struct obf_node **table, const char *key, const char *value, int age) +{ + struct obf_node **p; + for (p = table + hash(key); *p; p = &(*p)->next) { + int cmp = strcasecmp((*p)->key, key); + if (cmp == 0) + return; + if (cmp > 0) + break; + } + *p = new_node(key, value, age, *p); +} + +enum obf_flags { OBF_NONE = 0, OBF_ADD_IF_NOT_THERE = 1<<0, OBF_PATTERN_MODE = 1<<1, OBF_PARTIAL = 1<<2 }; + +static const char *obfuscate_component(const char *component, enum obf_flags flags) +{ + const char *obf_component; + + /* If we're in pattern mode and the pattern contains a wildcard character ('*' or '?') + * sending an obfuscated pattern is not going to work. The best we can do is send + * "*" for this component, which may end up matching too much, but it's better than + * not matching anything. If the pattern contains "**" then we need to preserve that. + */ + if (flags & OBF_PATTERN_MODE && (strchr(component, '*') || strchr(component, '?'))) { + if (strstr(component, "**")) + return "**"; + else + return "*"; + } + + obf_component = lookup_key(obf_table, component); + if (obf_component) + return obf_component; + + if (!(flags & OBF_ADD_IF_NOT_THERE)) + return component; + + // Component is not already in the obf_table so create it. + + obf_table_changed = 1; + char *s = new_array(char, OBF_KEY_LENGTH + 1); + char *c = strdup(component); + do { + obfuscate(s); + } while(contains_key(deobf_obf_table, s)); + + if (verbose > 2) + rprintf(FINFO, "obf inserting key %s, value %s\n", c, s); + + insert_key_value(obf_table, c, s, 0); + insert_key_value(deobf_obf_table, s, c, 0); + return s; +} + +static const char *deobfuscate_component(const char *obf_component, enum obf_flags flags) +{ + const char *s = lookup_key(deobf_obf_table, obf_component); + if (s) + /* deobfuscated component was found, so return it */ + return s; + else + /* not found, so assume that it is already deobfuscated */ + return obf_component; +} + +static void convert(const char *(*f)(const char *, enum obf_flags), char *fname_out, int size, const char *fname_in, enum obf_flags flags) +{ + char *tmp = strdup(fname_in), *tmp_p = tmp, *out_p = fname_out; + int first = 1, startslash, endslash, obfuscate_now; + + obfuscate_now = !(flags & OBF_PARTIAL); + + /* Patterns may start with "+ " or "- ", which we need to pass through unobfuscated. + */ + if (flags & OBF_PATTERN_MODE + && tmp_p + && strlen(tmp_p) >= 2 + && (tmp_p[0] == '+' || tmp_p[0] == '-') + && tmp_p[1] == ' ' + && size >= 2) { + *(out_p++) = tmp_p[0]; + *(out_p++) = ' '; + size -= 2; + tmp_p += 2; + } + startslash = tmp_p[0] == '/'; + endslash = tmp_p[strlen(tmp_p) - 1] == '/'; + + if (startslash && size > 0) { + *(out_p++) = '/'; + --size; + } + + for (tmp_p = strtok(tmp_p, "/"); tmp_p && size > 0; tmp_p = strtok(NULL, "/")) { + + /* If in partial mode then check whether we've found the special marker "!!!" which + * tells us that it's time to start obfuscating */ + if (flags & OBF_PARTIAL && !obfuscate_now && strcasecmp(tmp_p, OBF_ARGPATH_START_MARKER) == 0) { + obfuscate_now = 1; + continue; + } + + const char *conv = obfuscate_now ? f(tmp_p, flags) : tmp_p; + int len = strlen(conv); + if (!first) { + *(out_p++) = '/'; + --size; + } + strncpy(out_p, conv, size); + out_p += len; + size -= len; + first = 0; + } + if (endslash && size > 0) + *(out_p++) = '/'; + + *out_p = '\0'; + free(tmp); +} + +static void obfuscate_fname(char *fname_obf, int size, const char *fname) +{ + convert(obfuscate_component, fname_obf, size, fname, OBF_ADD_IF_NOT_THERE); +} + +void deobfuscate_fname(char *fname, int size, const char *fname_obf) +{ + convert(deobfuscate_component, fname, size, fname_obf, OBF_NONE); +} + +void obfuscate_pattern(char *pat_obf, int size, const char *pat) +{ + convert(obfuscate_component, pat_obf, size, pat, OBF_PATTERN_MODE | OBF_ADD_IF_NOT_THERE); +} + +void obfuscate_argpath(char *argpath_obf, int size, const char *argpath) +{ + convert(obfuscate_component, argpath_obf, size, argpath, OBF_PARTIAL); + if (verbose > 2) + rprintf(FINFO, "obfuscate_argpath(%s) = %s\n", argpath, argpath_obf); +} + +const char *obf_lookup_basename(const char *component) +{ + const char *obf_component = obfuscate_component(component, OBF_ADD_IF_NOT_THERE); + if (verbose > 2) + rprintf(FINFO, "obf_lookup_basename(%s) = %s\n", component, obf_component); + return obf_component; +} + +const char *obf_lookup_dirname(const char *dirname) +{ + const char *obf_dirname = lookup_key(obf_dirname_table, dirname); + if (!obf_dirname) { + char buf[MAXPATHLEN]; + obfuscate_fname(buf, MAXPATHLEN, dirname); + obf_dirname = strdup(buf); + insert_key_value(obf_dirname_table, dirname, obf_dirname, 0); + } + if (verbose > 2) + rprintf(FINFO, "obf_lookup_dirname(%s) = %s\n", dirname, obf_dirname); + return obf_dirname; +} + +const char *obf_lookup_basename(const char *component) +{ + return obfuscate_component(component, OBF_ADD_IF_NOT_THERE); +} + +const char *obf_lookup_dirname(const char *dirname) +{ + const char *obf_dirname = lookup_key(obf_dirname_table, dirname); + if (!obf_dirname) { + char buf[MAXPATHLEN]; + obfuscate_fname(buf, MAXPATHLEN, dirname); + obf_dirname = strdup(buf); + insert_key_value(obf_dirname_table, dirname, obf_dirname, 0); + } + return obf_dirname; +} + +int obf_load() +{ + int fd; + STRUCT_STAT st; + int32 version; + char fname[MAXPATHLEN], fname_obf[MAXPATHLEN]; + + /* + * We need to avoid obfuscating "." and "..", so add them to the table. + */ + char *dotdir = ".", *dotdotdir = ".."; + insert_key_value(obf_table, dotdir, dotdir, 0); + insert_key_value(deobf_obf_table, dotdir, dotdir, 0); + insert_key_value(obf_table, dotdotdir, dotdotdir, 0); + insert_key_value(deobf_obf_table, dotdotdir, dotdotdir, 0); + + /* + * Also avoid obfuscating "*" and "**", since these are used in filters. + */ + char *star = "*", *starstar = "**"; + insert_key_value(obf_table, star, star, 0); + insert_key_value(deobf_obf_table, star, star, 0); + insert_key_value(obf_table, starstar, starstar, 0); + insert_key_value(deobf_obf_table, starstar, starstar, 0); + + /* + * Also partial_dir which we use for storing partial transfers. + */ + if (partial_dir) + { + char *partial = strdup(partial_dir); + insert_key_value(obf_table, partial, partial, 0); + insert_key_value(deobf_obf_table, partial, partial, 0); + } + + fd = do_open(obfuscation_file, O_RDONLY, 0); + if (fd == -1) { + if (verbose > 0) + { + if (errno == ENOENT) + rprintf(FINFO, "obfuscation file %s: does not exist, a new file will be created\n", obfuscation_file); + else + rprintf(FINFO, "obfuscation file %s: could not open for loading, a new file will be created\n", obfuscation_file); + } + return 0; + } + + if (do_fstat(fd, &st) != 0) { + if (verbose > 0) + rprintf(FINFO, "obfuscation file %s: could not fstat\n", obfuscation_file); + close(fd); + return 0; + } + + if (st.st_size == 0) { + if (verbose > 0) + rprintf(FINFO, "obfuscation file %s: file is empty\n", obfuscation_file); + close(fd); + return 0; + } + + if (verbose > 0) + rprintf(FINFO, "obfuscation file %s: loading\n", obfuscation_file); + + + + version = read_int(fd); + if (version > OBF_VERSION) { + if (verbose > 0) + rprintf(FINFO, "obfuscation file %s: version %d > latest known version (%d), ignoring\n", obfuscation_file, version, OBF_VERSION); + return 0; + } + + while (1) { + char *f, *o; + int age = 0; + read_vstring(fd, fname, MAXPATHLEN); + if (strlen(fname) == 0) + break; // Empty string signifies that we've reached the end of the file. + + read_vstring(fd, fname_obf, MAXPATHLEN); + + if (version >= 1) + age = read_int(fd); + f = strdup(fname); + o = strdup(fname_obf); + age++; + insert_key_value(obf_table, f, o, age); + insert_key_value(deobf_obf_table, o, f, age); + } + close(fd); + if (verbose > 0) + rprintf(FINFO, "obfuscation file %s: loaded\n", obfuscation_file); + return 1; +} + +static void write_node(int fd, struct obf_node *p) +{ + if (p->age > OBF_MAX_AGE) { + if (verbose > 2) + rprintf(FINFO, "obf discarding key %s, value %s, age %d\n", p->key, p->value, p->age); + return; + } + if (verbose > 2) + rprintf(FINFO, "obf saving key %s, value %s, age %d\n", p->key, p->value, p->age); + write_vstring(fd, p->key, strlen(p->key)); + write_vstring(fd, p->value, strlen(p->value)); + write_int(fd, p->age); +} + +void obf_save(void) +{ + int fd; + unsigned i; + + char tmp[MAXPATHLEN]; + + if (!obfuscation_file || !obf_table_changed) + return; + + get_tmpname(tmp, obfuscation_file); + + fd = do_mkstemp(tmp, S_IRUSR|S_IWUSR); + if (fd == -1) { + if (verbose > 0) + rprintf(FINFO, "obfuscation file %s: could not open for saving\n", tmp); + return; + } + if (verbose > 2) + rprintf(FINFO, "obfuscation file %s: saving\n", tmp); + + write_int(fd, (int32)OBF_VERSION); + + for (i = 0; i < OBF_TABLE_SIZE; ++i) { + struct obf_node *p; + for (p = obf_table[i]; p; p = p->next) + write_node(fd, p); + } + + // Write an empty string to mark the end of file. + write_vstring(fd, "", 0); + + close(fd); + + if (verbose > 2) + rprintf(FINFO, "obfuscation file: renaming %s to %s\n", tmp, obfuscation_file); + + if (robust_rename(tmp, obfuscation_file, NULL, S_IRUSR|S_IWUSR) < 0) + rprintf(FERROR, "obfuscation file: could not rename %s to %s\n", tmp, obfuscation_file); + + return; +} diff -r 17ce0bc36a49 -r 87cb4f9014f1 Tools/Rsync/options.c --- a/Tools/Rsync/options.c Tue Feb 23 21:03:38 2010 +0000 +++ b/Tools/Rsync/options.c Fri Oct 29 12:32:07 2010 +1100 @@ -105,6 +105,7 @@ int safe_symlinks = 0; int copy_unsafe_links = 0; int size_only = 0; +int times_only = 0; int daemon_bwlimit = 0; int bwlimit = 0; int fuzzy_basis = 0; @@ -123,6 +124,8 @@ int delay_updates = 0; long block_size = 0; /* "long" because popt can't set an int32. */ char *skip_compress = NULL; +int backup_nt_streams = 0; +int restore_nt_streams = 0; /** Network address family. **/ int default_af_hint @@ -162,6 +165,12 @@ char *logfile_format = NULL; char *stdout_format = NULL; char *password_file = NULL; +char *source_filter = NULL; +char *dest_filter = NULL; +char *basis_filter = NULL; +char *file_size_cache = NULL; +char *obfuscation_file = NULL; +char *source_filter_tmp = "/tmp/"; char *rsync_path = RSYNC_PATH; char *backup_dir = NULL; char backup_dir_buf[MAXPATHLEN]; @@ -388,6 +397,7 @@ rprintf(F," --contimeout=SECONDS set daemon connection timeout in seconds\n"); rprintf(F," -I, --ignore-times don't skip files that match in size and mod-time\n"); rprintf(F," --size-only skip files that match in size\n"); + rprintf(F," --times-only skip files that match in mod-time\n"); rprintf(F," --modify-window=NUM compare mod-times with reduced accuracy\n"); rprintf(F," -T, --temp-dir=DIR create temporary files in directory DIR\n"); rprintf(F," -y, --fuzzy find similar file for basis if no dest file\n"); @@ -427,6 +437,11 @@ rprintf(F," --write-batch=FILE write a batched update to FILE\n"); rprintf(F," --only-write-batch=FILE like --write-batch but w/o updating destination\n"); rprintf(F," --read-batch=FILE read a batched update from FILE\n"); + rprintf(F," --source-filter=COMMAND filter file through COMMAND at source\n"); + rprintf(F," --dest-filter=COMMAND filter file through COMMAND at destination\n"); + rprintf(F," --basis-filter=COMMAND filter basis file through COMMAND at destination\n"); + rprintf(F," --file-size-cache=FILE caches sizes of sent files in FILE\n"); + rprintf(F," --source-filter-tmp=DIR use DIR instead of /tmp/ for temporary files created by --source-filter\n"); rprintf(F," --protocol=NUM force an older protocol version to be used\n"); #ifdef ICONV_OPTION rprintf(F," --iconv=CONVERT_SPEC request charset conversion of filenames\n"); @@ -530,6 +545,7 @@ {"chmod", 0, POPT_ARG_STRING, 0, OPT_CHMOD, 0, 0 }, {"ignore-times", 'I', POPT_ARG_NONE, &ignore_times, 0, 0, 0 }, {"size-only", 0, POPT_ARG_NONE, &size_only, 0, 0, 0 }, + {"times-only", 0, POPT_ARG_NONE, ×_only , 0, 0, 0 }, {"one-file-system", 'x', POPT_ARG_NONE, 0, 'x', 0, 0 }, {"no-one-file-system",'x',POPT_ARG_VAL, &one_file_system, 0, 0, 0 }, {"no-x", 'x', POPT_ARG_VAL, &one_file_system, 0, 0, 0 }, @@ -645,6 +661,12 @@ {"password-file", 0, POPT_ARG_STRING, &password_file, 0, 0, 0 }, {"blocking-io", 0, POPT_ARG_VAL, &blocking_io, 1, 0, 0 }, {"no-blocking-io", 0, POPT_ARG_VAL, &blocking_io, 0, 0, 0 }, + {"source-filter", 0, POPT_ARG_STRING, &source_filter, 0, 0, 0 }, + {"dest-filter", 0, POPT_ARG_STRING, &dest_filter, 0, 0, 0 }, + {"basis-filter", 0, POPT_ARG_STRING, &basis_filter, 0, 0, 0 }, + {"file-size-cache", 0, POPT_ARG_STRING, &file_size_cache, 0, 0, 0 }, + {"obfuscation-file", 0, POPT_ARG_STRING, &obfuscation_file, 0, 0, 0 }, + {"source-filter-tmp",0, POPT_ARG_STRING, &source_filter_tmp, 0, 0, 0 }, {"protocol", 0, POPT_ARG_INT, &protocol_version, 0, 0, 0 }, {"checksum-seed", 0, POPT_ARG_INT, &checksum_seed, 0, 0, 0 }, {"server", 0, POPT_ARG_NONE, 0, OPT_SERVER, 0, 0 }, @@ -654,6 +676,8 @@ {"daemon", 0, POPT_ARG_NONE, 0, OPT_DAEMON, 0, 0 }, {"detach", 0, POPT_ARG_NONE, 0, OPT_DAEMON, 0, 0 }, {"no-detach", 0, POPT_ARG_NONE, 0, OPT_DAEMON, 0, 0 }, + {"backup-nt-streams",0, POPT_ARG_VAL, &backup_nt_streams, 1, 0, 0 }, + {"restore-nt-streams",0, POPT_ARG_VAL, &restore_nt_streams, 1, 0, 0 }, {0,0,0,0, 0, 0, 0} }; @@ -1967,12 +1991,51 @@ args[ac++] = "--super"; if (size_only) args[ac++] = "--size-only"; + if (restore_nt_streams) + args[ac++] = "--restore-nt-streams"; } else { if (skip_compress) { if (asprintf(&arg, "--skip-compress=%s", skip_compress) < 0) goto oom; args[ac++] = arg; } + if (backup_nt_streams) + args[ac++] = "--backup-nt-streams"; + } + + if (times_only && am_sender) + args[ac++] = "--times-only"; + + if (source_filter && !am_sender) { + /* Need to single quote the arg to keep the remote shell + * from splitting it. FIXME: breaks if command has single quotes. */ + if (asprintf(&arg, "--source-filter='%s'", source_filter) < 0) + goto oom; + args[ac++] = arg; + } + + if (dest_filter && am_sender) { + /* Need to single quote the arg to keep the remote shell + * from splitting it. FIXME: breaks if command has single quotes. */ + if (asprintf(&arg, "--dest-filter='%s'", dest_filter) < 0) + goto oom; + args[ac++] = arg; + } + + if (basis_filter && am_sender) { + /* Need to single quote the arg to keep the remote shell + * from splitting it. FIXME: breaks if command has single quotes. */ + if (asprintf(&arg, "--basis-filter='%s'", basis_filter) < 0) + goto oom; + args[ac++] = arg; + } + + if (source_filter_tmp && source_filter && !am_sender) { + /* Need to single quote the arg to keep the remote shell + * from splitting it. FIXME: breaks if command has single quotes. */ + if (asprintf(&arg, "--source-filter-tmp='%s'", source_filter_tmp) < 0) + goto oom; + args[ac++] = arg; } if (modify_window_set) { diff -r 17ce0bc36a49 -r 87cb4f9014f1 Tools/Rsync/pipe.c --- a/Tools/Rsync/pipe.c Tue Feb 23 21:03:38 2010 +0000 +++ b/Tools/Rsync/pipe.c Fri Oct 29 12:32:07 2010 +1100 @@ -167,3 +167,77 @@ return pid; } + +pid_t run_filter(char *command[], int out, int *pipe_to_filter) +{ + pid_t pid; + int pipefds[2]; + + if (verbose >= 2) + print_child_argv("opening connection using:", command); + + if (pipe(pipefds) < 0) { + rsyserr(FERROR, errno, "pipe"); + exit_cleanup(RERR_IPC); + } + + pid = fork(); + if (pid == -1) { + rsyserr(FERROR, errno, "fork"); + exit_cleanup(RERR_IPC); + } + + if (pid == 0) { + if (dup2(pipefds[0], STDIN_FILENO) < 0 + || close(pipefds[1]) < 0 + || dup2(out, STDOUT_FILENO) < 0) { + rsyserr(FERROR, errno, "Failed dup/close"); + exit_cleanup(RERR_IPC); + } + umask(orig_umask); + set_blocking(STDIN_FILENO); + if (blocking_io) + set_blocking(STDOUT_FILENO); + execvp(command[0], command); + rsyserr(FERROR, errno, "Failed to exec %s", command[0]); + exit_cleanup(RERR_IPC); + } + + if (close(pipefds[0]) < 0) { + rsyserr(FERROR, errno, "Failed to close"); + exit_cleanup(RERR_IPC); + } + + *pipe_to_filter = pipefds[1]; + + return pid; +} + +pid_t run_filter_on_file(char *command[], int out, int in) +{ + pid_t pid; + + if (verbose >= 2) + print_child_argv("opening connection using:", command); + + pid = fork(); + if (pid == -1) { + rsyserr(FERROR, errno, "fork"); + exit_cleanup(RERR_IPC); + } + + if (pid == 0) { + if ((in >= 0 && dup2(in, STDIN_FILENO) < 0) + || (out >= 0 && dup2(out, STDOUT_FILENO)) < 0) { + rsyserr(FERROR, errno, "Failed to dup2"); + exit_cleanup(RERR_IPC); + } + if (blocking_io) + set_blocking(STDOUT_FILENO); + execvp(command[0], command); + rsyserr(FERROR, errno, "Failed to exec %s", command[0]); + exit_cleanup(RERR_IPC); + } + + return pid; +} diff -r 17ce0bc36a49 -r 87cb4f9014f1 Tools/Rsync/proto.h --- a/Tools/Rsync/proto.h Tue Feb 23 21:03:38 2010 +0000 +++ b/Tools/Rsync/proto.h Fri Oct 29 12:32:07 2010 +1100 @@ -85,7 +85,7 @@ int link_stat(const char *path, STRUCT_STAT *stp, int follow_dirlinks); int change_pathname(struct file_struct *file, const char *dir, int dirlen); struct file_struct *make_file(const char *fname, struct file_list *flist, - STRUCT_STAT *stp, int flags, int filter_level); + STRUCT_STAT *stp, int flags, int filter_level, const char *real_fname); void unmake_file(struct file_struct *file); void send_extra_file_list(int f, int at_least); struct file_list *send_file_list(int f, int argc, char *argv[]); @@ -100,7 +100,14 @@ int f_name_has_prefix(const struct file_struct *f1, const struct file_struct *f2); char *f_name_buf(void); char *f_name(const struct file_struct *f, char *fbuf); +char *f_name_obf(const struct file_struct *f, char *fbuf); +char *f_name_maybe_obf(const struct file_struct *f, char *fbuf); struct file_list *get_dirlist(char *dirname, int dlen, int ignore_filter_rules); +void fscache_clear(void); +int fscache_lookup(const char *fname, int64 *pSize); +void fscache_insert(const char *fname, int64 size); +int fscache_load(); +void fscache_save(void); int unchanged_attrs(const char *fname, struct file_struct *file, stat_x *sxp); void itemize(const char *fnamecmp, struct file_struct *file, int ndx, int statret, stat_x *sxp, int32 iflags, uchar fnamecmp_type, @@ -241,6 +248,7 @@ void log_delete(const char *fname, int mode); void log_exit(int code, const char *file, int line); pid_t wait_process(pid_t pid, int *status_ptr, int flags); +void wait_process_with_flush(pid_t pid, int *exit_code_ptr); int child_main(int argc, char *argv[]); void start_server(int f_in, int f_out, int argc, char *argv[]); int client_run(int f_in, int f_out, pid_t pid, int argc, char *argv[]); @@ -249,6 +257,26 @@ int main(int argc,char *argv[]); void match_sums(int f, struct sum_struct *s, struct map_struct *buf, OFF_T len); void match_report(void); +int ntfs_streams_to_fd(const char *path, int fd); +int fd_to_ntfs_streams(int fd, const char *path); +void ntstreams_dirname(char *dest, size_t destsize, const char *dirname); +size_t ntstreams_filename(char *dest, char *orig, size_t destsize, const char *dirname, const char *basename); +int ntstreams_orig_filename(char *dest, size_t destsize, const char *dirname, const char *basename); +char *ntstreams_get_last_error_message(); +const char *ntstreams_get_last_error_what(); +int set_privilege_backup(int bEnablePrivilege); +int set_privilege_restore(int bEnablePrivilege); +int set_privilege_security(int bEnablePrivilege); +int set_privilege_take_ownership(int bEnablePrivilege); +int set_privilege(const char *lpszPrivilege, int bEnablePrivilege); +void ntstreams_restore_update(const char *fname, const struct file_struct *file); +void deobfuscate_fname(char *fname, int size, const char *fname_obf); +void obfuscate_pattern(char *pat_obf, int size, const char *pat); +void obfuscate_argpath(char *argpath_obf, int size, const char *argpath); +const char *obf_lookup_basename(const char *component); +const char *obf_lookup_dirname(const char *dirname); +int obf_load(); +void obf_save(void); void usage(enum logcode F); void option_error(void); int parse_arguments(int *argc_p, const char ***argv_p); @@ -260,6 +288,8 @@ pid_t piped_child(char **command, int *f_in, int *f_out); pid_t local_child(int argc, char **argv, int *f_in, int *f_out, int (*child_main)(int, char*[])); +pid_t run_filter(char *command[], int out, int *pipe_to_filter); +pid_t run_filter_on_file(char *command[], int out, int in); void set_current_file_index(struct file_struct *file, int ndx); void end_progress(OFF_T size); void show_progress(OFF_T ofs, OFF_T size); @@ -378,6 +408,16 @@ int flist_ndx_pop(flist_ndx_list *lp); void *expand_item_list(item_list *lp, size_t item_size, const char *desc, int incr); +char *my_strtok(char *s); +void tokenize_filter_args( + /* OUT */ char *filter_argv[], + /* IN/OUT */ char *filter, + /* IN */ const char *filter_type); +enum filter_args_info substitute_filter_args( + /* OUT */ char *filter_argv_subst[], + /* IN */ char *source_fname, + /* IN */ char *dest_fname, + /* IN */ char *filter_argv[]); void free_xattr(stat_x *sxp); int get_xattr(const char *fname, stat_x *sxp); int copy_xattrs(const char *source, const char *dest); diff -r 17ce0bc36a49 -r 87cb4f9014f1 Tools/Rsync/receiver.c --- a/Tools/Rsync/receiver.c Tue Feb 23 21:03:38 2010 +0000 +++ b/Tools/Rsync/receiver.c Fri Oct 29 12:32:07 2010 +1100 @@ -52,10 +52,12 @@ extern mode_t orig_umask; extern struct stats stats; extern char *tmpdir; +extern char *dest_filter; extern char *partial_dir; extern char *basis_dir[]; extern struct file_list *cur_flist, *first_flist, *dir_flist; extern struct filter_list_struct daemon_filter_list; +extern int restore_nt_streams; static struct bitbag *delayed_bits = NULL; static int phase = 0, redoing = 0; @@ -422,12 +424,13 @@ * Receiver process runs on the same host as the generator process. */ int recv_files(int f_in, char *local_name) { - int fd1,fd2; + int fd1,fd2,fd2_dest_filter = -1; STRUCT_STAT st; int iflags, xlen; char *fname, fbuf[MAXPATHLEN]; char xname[MAXPATHLEN]; char fnametmp[MAXPATHLEN]; + char fnametmp_dest_filter[MAXPATHLEN]; char *fnamecmp, *partialptr; char fnamecmpbuf[MAXPATHLEN]; uchar fnamecmp_type; @@ -441,6 +444,10 @@ const char *parent_dirname = ""; #endif int ndx, recv_ok; + pid_t pid = 0; + char *filter_argv[MAX_FILTER_ARGS + 1]; + char *filter_argv_subst[MAX_FILTER_ARGS + 1]; + enum filter_args_info fai = fai_has_none; if (verbose > 2) rprintf(FINFO, "recv_files(%d) starting\n", cur_flist->used); @@ -448,7 +455,25 @@ if (delay_updates) delayed_bits = bitbag_create(cur_flist->used + 1); + if (dest_filter) + tokenize_filter_args(filter_argv, dest_filter, "dest"); + + if (restore_nt_streams) { + if (verbose >2) + rprintf(FINFO, "requesting SeRestorePrivilege\n"); + if (!set_privilege_restore(1)) + rprintf(FERROR, "could not obtain SeRestorePrivilege: %s\n", + ntstreams_get_last_error_message()); + if (verbose >2) + rprintf(FINFO, "requesting SeSecurityPrivilege\n"); + if (!set_privilege_security(1)) + rprintf(FERROR, "could not obtain SeSecurityPrivilege: %s\n", + ntstreams_get_last_error_message()); + } + while (1) { + int unlink_basis_filter_tmp = 0; + fai = fai_has_none; cleanup_disable(); /* This call also sets cur_flist. */ @@ -482,6 +507,9 @@ if (verbose > 2) rprintf(FINFO, "recv_files(%s)\n", fname); + if (restore_nt_streams) + ntstreams_restore_update(fname, file); + #ifdef SUPPORT_XATTRS if (iflags & ITEM_REPORT_XATTR && do_xfers) recv_xattr_request(file, f_in); @@ -578,6 +606,10 @@ case FNAMECMP_BACKUP: fnamecmp = get_backup_name(fname); break; + case FNAMECMP_FILTERED: + unlink_basis_filter_tmp = 1; + fnamecmp = xname; + break; case FNAMECMP_FUZZY: if (file->dirname) { pathjoin(fnamecmpbuf, MAXPATHLEN, @@ -692,6 +724,7 @@ /* We now check to see if we are writing the file "inplace" */ if (inplace) { fd2 = do_open(fname, O_WRONLY|O_CREAT, 0600); + strncpy(fnametmp, fname, MAXPATHLEN); if (fd2 == -1) { rsyserr(FERROR_XFER, errno, "open %s failed", full_fname(fname)); @@ -717,6 +750,33 @@ else if (!am_server && verbose && do_progress) rprintf(FINFO, "%s\n", fname); + if (dest_filter) { + int tmpfd = open_tmpfile(fnametmp_dest_filter, fname, file); + fai = substitute_filter_args(filter_argv_subst, fnametmp_dest_filter, fnametmp, filter_argv); + if (fai == fai_has_none) { + pid = run_filter(filter_argv, fd2, &fd2); + if (tmpfd != -1) + { + close(tmpfd); + unlink(fnametmp_dest_filter); + } + fd2_dest_filter = -1; + } else { + if (tmpfd == -1) { + discard_receive_data(f_in, F_LENGTH(file)); + if (fd1 != -1) + close(fd1); + if (fd2 != -1) + close(fd2); + if (inc_recurse) + send_msg_int(MSG_NO_SEND, ndx); + continue; + } + fd2_dest_filter = fd2; + fd2 = tmpfd; + } + } + /* recv file data */ recv_ok = receive_data(f_in, fnamecmp, fd1, st.st_size, fname, fd2, F_LENGTH(file)); @@ -725,12 +785,46 @@ if (fd1 != -1) close(fd1); - if (close(fd2) < 0) { + + if (fd2 >= 0 && close(fd2) < 0) { rsyserr(FERROR, errno, "close failed on %s", full_fname(fnametmp)); exit_cleanup(RERR_FILEIO); } + if (dest_filter && fai != fai_has_none) { + int status; + if (fai & fai_has_source) { + fd2 = -1; + } else { + fd2 = do_open(fnametmp_dest_filter, O_RDONLY, 0); + } + if (fai & fai_has_dest) { + close(fd2_dest_filter); + fd2_dest_filter = -1; + } + pid = run_filter_on_file(filter_argv_subst, fd2_dest_filter, fd2); + if (fd2 >= 0) close(fd2); + if (fd2_dest_filter >= 0) close(fd2_dest_filter); + wait_process_with_flush(pid, &status); + if (status != 0) { + rprintf(FERROR, "dest-filter %s exited code: %d\n", + dest_filter, status); + continue; + } + unlink(fnametmp_dest_filter); + } + + if (dest_filter && fai == fai_has_none) { + int status; + wait_process_with_flush(pid, &status); + if (status != 0) { + rprintf(FERROR, "filter %s exited code: %d\n", + dest_filter, status); + continue; + } + } + if ((recv_ok && (!delay_updates || !partialptr)) || inplace) { if (partialptr == fname) partialptr = NULL; @@ -760,6 +854,9 @@ } else do_unlink(fnametmp); + if (unlink_basis_filter_tmp) + do_unlink(fnamecmp); + cleanup_disable(); if (read_batch) @@ -811,6 +908,9 @@ break; } } + if (restore_nt_streams) + ntstreams_restore_update(NULL, NULL); + if (make_backups < 0) make_backups = -make_backups; diff -r 17ce0bc36a49 -r 87cb4f9014f1 Tools/Rsync/rsync.1 --- a/Tools/Rsync/rsync.1 Tue Feb 23 21:03:38 2010 +0000 +++ b/Tools/Rsync/rsync.1 Fri Oct 29 12:32:07 2010 +1100 @@ -461,6 +461,7 @@ \-\-contimeout=SECONDS set daemon connection timeout in seconds \-I, \-\-ignore\-times don'\&t skip files that match size and time \-\-size\-only skip files that match in size + \-\-times\-only skip files that match in mod-time \-\-modify\-window=NUM compare mod-times with reduced accuracy \-T, \-\-temp\-dir=DIR create temporary files in directory DIR \-y, \-\-fuzzy find similar file for basis if no dest file @@ -500,6 +501,8 @@ \-\-write\-batch=FILE write a batched update to FILE \-\-only\-write\-batch=FILE like \-\-write\-batch but w/o updating dest \-\-read\-batch=FILE read a batched update from FILE + \-\-source\-filter=COMMAND filter file through COMMAND at source + \-\-dest\-filter=COMMAND filter file through COMMAND at destination \-\-protocol=NUM force an older protocol version to be used \-\-iconv=CONVERT_SPEC request charset conversion of filenames \-\-checksum\-seed=NUM set block/file checksum seed (advanced) @@ -2392,6 +2395,35 @@ If \fIFILE\fP is \fB\-\fP, the batch data will be read from standard input. See the \(dq\&BATCH MODE\(dq\& section for details. .IP +.IP "\fB\-\-source\-filter=COMMAND\fP" +This option allows the user to specify a +filter program that will be applied to the contents of all transferred +regular files before the data is sent to destination. COMMAND will receive +the data on its standard input and it should write the filtered data to +standard output. COMMAND should exit non-zero if it cannot process the +data or if it encounters an error when writing the data to stdout. +.IP +Example: \-\-source\-filter=\(dq\&gzip \-9\(dq\& will cause remote files to be +compressed. +Use of \-\-source\-filter automatically enables \-\-whole\-file. +If your filter does not output the same number of bytes that it received on +input, you should use \-\-times\-only to disable size and content checks on +subsequent rsync runs. +.IP +.IP "\fB\-\-dest\-filter=COMMAND\fP" +This option allows you to specify a filter +program that will be applied to the contents of all transferred regular +files before the data is written to disk. COMMAND will receive the data on +its standard input and it should write the filtered data to standard +output. COMMAND should exit non-zero if it cannot process the data or if +it encounters an error when writing the data to stdout. +.IP +Example: \-\-dest\-filter=\(dq\&gzip \-9\(dq\& will cause remote files to be compressed. +Use of \-\-dest\-filter automatically enables \-\-whole\-file. +If your filter does not output the same number of bytes that it +received on input, you should use \-\-times\-only to disable size and +content checks on subsequent rsync runs. +.IP .IP "\fB\-\-protocol=NUM\fP" Force an older protocol version to be used. This is useful for creating a batch file that is compatible with an older diff -r 17ce0bc36a49 -r 87cb4f9014f1 Tools/Rsync/rsync.h --- a/Tools/Rsync/rsync.h Tue Feb 23 21:03:38 2010 +0000 +++ b/Tools/Rsync/rsync.h Fri Oct 29 12:32:07 2010 +1100 @@ -81,6 +81,10 @@ #define FLAG_SKIP_GROUP (1<<10) /* receiver/generator */ #define FLAG_TIME_FAILED (1<<11)/* generator */ +/* NT Streams */ +#define FLAG_NT_STREAM (1<<12) /* sender */ +#define FLAG_NT_STREAM_DIR (1<<13) /* sender */ + /* These flags are passed to functions but not stored. */ #define FLAG_DIVERT_DIRS (1<<16)/* sender */ @@ -138,6 +142,7 @@ #define IOERR_DEL_LIMIT (1<<2) #define MAX_ARGS 1000 +#define MAX_FILTER_ARGS 100 #define MAX_BASIS_DIRS 20 #define MAX_SERVER_ARGS (MAX_BASIS_DIRS*2 + 100) @@ -169,6 +174,7 @@ #define FNAMECMP_PARTIAL_DIR 0x81 #define FNAMECMP_BACKUP 0x82 #define FNAMECMP_FUZZY 0x83 +#define FNAMECMP_FILTERED 0x84 /* For use by the itemize_changes code */ #define ITEM_REPORT_ATIME (1<<0) @@ -626,6 +632,8 @@ uint32 len32; /* Lowest 32 bits of the file's length */ uint16 mode; /* The item's type and permissions */ uint16 flags; /* The FLAG_* bits for this item */ + const char *dirname_obf; /* Obfuscated dir name */ + const char *basename_obf; /* Obfuscated basename */ const char basename[1]; /* The basename (AKA filename) follows */ }; @@ -913,6 +921,8 @@ #define ACL_READY(sx) ((sx).acc_acl != NULL) #define XATTR_READY(sx) ((sx).xattr != NULL) +enum filter_args_info { fai_has_none = 0, fai_has_source = 1, fai_has_dest = 2 }; + #include "proto.h" #ifndef SUPPORT_XATTRS @@ -1145,3 +1155,5 @@ #ifdef MAINTAINER_MODE const char *get_panic_action(void); #endif + +#define OBF_ARGPATH_START_MARKER "!!!" diff -r 17ce0bc36a49 -r 87cb4f9014f1 Tools/Rsync/rsync.yo --- a/Tools/Rsync/rsync.yo Tue Feb 23 21:03:38 2010 +0000 +++ b/Tools/Rsync/rsync.yo Fri Oct 29 12:32:07 2010 +1100 @@ -386,6 +386,7 @@ --contimeout=SECONDS set daemon connection timeout in seconds -I, --ignore-times don't skip files that match size and time --size-only skip files that match in size + --times-only skip files that match in mod-time --modify-window=NUM compare mod-times with reduced accuracy -T, --temp-dir=DIR create temporary files in directory DIR -y, --fuzzy find similar file for basis if no dest file @@ -425,6 +426,8 @@ --write-batch=FILE write a batched update to FILE --only-write-batch=FILE like --write-batch but w/o updating dest --read-batch=FILE read a batched update from FILE + --source-filter=COMMAND filter file through COMMAND at source + --dest-filter=COMMAND filter file through COMMAND at destination --protocol=NUM force an older protocol version to be used --iconv=CONVERT_SPEC request charset conversion of filenames --checksum-seed=NUM set block/file checksum seed (advanced) @@ -2084,6 +2087,33 @@ If em(FILE) is bf(-), the batch data will be read from standard input. See the "BATCH MODE" section for details. +dit(bf(--source-filter=COMMAND)) This option allows the user to specify a +filter program that will be applied to the contents of all transferred +regular files before the data is sent to destination. COMMAND will receive +the data on its standard input and it should write the filtered data to +standard output. COMMAND should exit non-zero if it cannot process the +data or if it encounters an error when writing the data to stdout. + +Example: --source-filter="gzip -9" will cause remote files to be +compressed. +Use of --source-filter automatically enables --whole-file. +If your filter does not output the same number of bytes that it received on +input, you should use --times-only to disable size and content checks on +subsequent rsync runs. + +dit(bf(--dest-filter=COMMAND)) This option allows you to specify a filter +program that will be applied to the contents of all transferred regular +files before the data is written to disk. COMMAND will receive the data on +its standard input and it should write the filtered data to standard +output. COMMAND should exit non-zero if it cannot process the data or if +it encounters an error when writing the data to stdout. + +Example: --dest-filter="gzip -9" will cause remote files to be compressed. +Use of --dest-filter automatically enables --whole-file. +If your filter does not output the same number of bytes that it +received on input, you should use --times-only to disable size and +content checks on subsequent rsync runs. + dit(bf(--protocol=NUM)) Force an older protocol version to be used. This is useful for creating a batch file that is compatible with an older version of rsync. For instance, if rsync 2.6.4 is being used with the diff -r 17ce0bc36a49 -r 87cb4f9014f1 Tools/Rsync/sender.c --- a/Tools/Rsync/sender.c Tue Feb 23 21:03:38 2010 +0000 +++ b/Tools/Rsync/sender.c Fri Oct 29 12:32:07 2010 +1100 @@ -42,8 +42,12 @@ extern int inplace; extern int batch_fd; extern int write_batch; +extern char *source_filter; +extern char *source_filter_tmp; +extern char *file_size_cache; extern struct stats stats; extern struct file_list *cur_flist, *first_flist, *dir_flist; +extern int backup_nt_streams; /** * @file @@ -163,7 +167,8 @@ struct sum_struct *s; struct map_struct *mbuf = NULL; STRUCT_STAT st; - char fname[MAXPATHLEN], xname[MAXPATHLEN]; + char fname[MAXPATHLEN], xname[MAXPATHLEN], ntstreams_tmpname[MAXPATHLEN]; + char *fname_to_open; const char *path, *slash; uchar fnamecmp_type; int iflags, xlen; @@ -175,10 +180,29 @@ int f_xfer = write_batch < 0 ? batch_fd : f_out; int save_io_error = io_error; int ndx, j; + char *filter_argv[MAX_FILTER_ARGS + 1]; + char tmp[MAXPATHLEN]; + int unlink_tmp = 0, unlink_ntstreams_tmp = 0; + + if (source_filter) + tokenize_filter_args(filter_argv, source_filter, "source"); if (verbose > 2) rprintf(FINFO, "send_files starting\n"); + if (backup_nt_streams) { + if (verbose >2) + rprintf(FINFO, "requesting SeBackupPrivilege\n"); + if (!set_privilege_backup(1)) + rprintf(FERROR, "could not obtain SeBackupPrivilege: %s\n", + ntstreams_get_last_error_message()); + if (verbose >2) + rprintf(FINFO, "requesting SeSecurityPrivilege\n"); + if (!set_privilege_security(1)) + rprintf(FERROR, "could not obtain SeSecurityPrivilege: %s\n", + ntstreams_get_last_error_message()); + } + while (1) { if (inc_recurse) send_extra_file_list(f_out, FILECNT_LOOKAHEAD); @@ -218,6 +242,7 @@ if (!change_pathname(file, NULL, 0)) continue; f_name(file, fname); + fname_to_open = fname; if (verbose > 2) rprintf(FINFO, "send_files(%d, %s%s%s)\n", ndx, path,slash,fname); @@ -279,14 +304,41 @@ exit_cleanup(RERR_PROTOCOL); } - fd = do_open(fname, O_RDONLY, 0); + unlink_ntstreams_tmp = 0; + if (backup_nt_streams && file->flags & FLAG_NT_STREAM) { + int fd2; + char orig_fname[MAXPATHLEN]; + ntstreams_orig_filename(orig_fname, MAXPATHLEN, file->dirname, file->basename); + if (verbose > 2) { + rprintf(FINFO, "reading nt streams for %s\n", orig_fname); + } + pathjoin(ntstreams_tmpname, MAXPATHLEN, source_filter_tmp, "rsync-nt_streamsXXXXXX"); + fd2 = mkstemp(ntstreams_tmpname); + if (fd2 == -1) { + rprintf(FERROR, "mkstemp %s failed, skipping nt streams: %s\n", + orig_fname, strerror(errno)); + continue; + } + if (ntfs_streams_to_fd(orig_fname, fd2) <= 0) { + rprintf(FERROR, "ntfs_streams_to_fd %s failed on %s, skipping nt streams: %s\n", + ntstreams_get_last_error_what(), orig_fname, ntstreams_get_last_error_message()); + close(fd2); + continue; + } + close(fd2); + unlink_ntstreams_tmp = 1; + fname_to_open = ntstreams_tmpname; + } + + unlink_tmp = 0; + fd = do_open(fname_to_open, O_RDONLY, 0); if (fd == -1) { if (errno == ENOENT) { enum logcode c = am_daemon && protocol_version < 28 ? FERROR : FWARNING; io_error |= IOERR_VANISHED; - rprintf(c, "file has vanished: %s\n", + rprintf(c, "send_files: file has vanished: %s\n", full_fname(fname)); } else { io_error |= IOERR_GENERAL; @@ -300,6 +352,41 @@ continue; } + if (source_filter) { + int fd2; + pathjoin(tmp, MAXPATHLEN, source_filter_tmp, "rsync-filtered_sourceXXXXXX"); + fd2 = mkstemp(tmp); + if (fd2 == -1) { + rprintf(FERROR, "mkstemp %s failed: %s\n", + tmp, strerror(errno)); + } else { + int status; + char *filter_argv_subst[MAX_FILTER_ARGS + 1]; + enum filter_args_info fai = substitute_filter_args(filter_argv_subst, fname_to_open, tmp, filter_argv); + if (fai & fai_has_source) { + close(fd); + fd = -1; + } + if (fai & fai_has_dest) { + close(fd2); + fd2 = -1; + } + pid_t pid = run_filter_on_file(filter_argv_subst, fd2, fd); + if (fd >= 0) close(fd); + if (fd2 >= 0) close(fd2); + wait_process_with_flush(pid, &status); + if (status != 0) { + rprintf(FERROR, + "bypassing source filter %s; exited with code: %d\n", + source_filter, status); + fd = do_open(fname_to_open, O_RDONLY, 0); + } else { + fd = do_open(tmp, O_RDONLY, 0); + unlink_tmp = 1; + } + } + } + /* map the local file */ if (do_fstat(fd, &st) != 0) { io_error |= IOERR_GENERAL; @@ -309,6 +396,8 @@ exit_cleanup(RERR_PROTOCOL); } + fscache_insert(fname, st.st_size); + if (st.st_size) { int32 read_size = MAX(s->blength * 3, MAX_MAP_SIZE); mbuf = map_file(fd, st.st_size, read_size, s->blength); @@ -350,6 +439,12 @@ } } close(fd); + if (unlink_tmp) + unlink(tmp); + *tmp = '\0'; + + if (unlink_ntstreams_tmp) + unlink(ntstreams_tmpname); free_sums(s); diff -r 17ce0bc36a49 -r 87cb4f9014f1 Tools/Rsync/util.c --- a/Tools/Rsync/util.c Tue Feb 23 21:03:38 2010 +0000 +++ b/Tools/Rsync/util.c Fri Oct 29 12:32:07 2010 +1100 @@ -455,6 +455,7 @@ static int num_pids; /** Fork and record the pid of the child. **/ +/** NOTE: don't call this more than 10 times!!! - david.overton **/ pid_t do_fork(void) { pid_t newpid = fork(); @@ -1690,3 +1691,73 @@ } return (char*)lp->items + (lp->count++ * item_size); } + +char *my_strtok(char *s) +{ + static char *p; + char *tok; + int in_quote = 0; + if (s) + p = s; + if (!p || !*p) + return NULL; + + if (*p == '\'' || *p == '"') { + in_quote = 1; + *p++ = '\0'; + } + for (tok = p; *p; ++p) { + if ((!in_quote && (*p == ' ' || *p == '\t')) || (in_quote && (*p == '\'' || *p == '"'))) { + // We found the end of this token so clobber any remaining spaces and return the token. + do { + *p++ = '\0'; + } while (*p == ' ' || *p == '\t'); + break; + } + } + return tok; +} + +void tokenize_filter_args( + /* OUT */ char *filter_argv[], + /* IN/OUT */ char *filter, + /* IN */ const char *filter_type) +{ + char *p; + int i; + for (p = my_strtok(filter), i = 0; + p && i < MAX_FILTER_ARGS; + p = my_strtok(0)) + filter_argv[i++] = p; + filter_argv[i] = NULL; + if (p) { + rprintf(FERROR, + "Too many arguments to %s-filter (> %d)\n", + filter_type, MAX_FILTER_ARGS); + exit_cleanup(RERR_SYNTAX); + } +} + +enum filter_args_info substitute_filter_args( + /* OUT */ char *filter_argv_subst[], + /* IN */ char *source_fname, + /* IN */ char *dest_fname, + /* IN */ char *filter_argv[]) +{ + int i; + enum filter_args_info fai = fai_has_none; + + for (i = 0; filter_argv[i] && i < MAX_FILTER_ARGS; ++i) { + if (strcmp(filter_argv[i], "%s") == 0) { + filter_argv_subst[i] = source_fname; + fai |= fai_has_source; + } else if (strcmp(filter_argv[i], "%d") == 0) { + filter_argv_subst[i] = dest_fname; + fai |= fai_has_dest; + } else { + filter_argv_subst[i] = filter_argv[i]; + } + } + filter_argv_subst[i] = NULL; + return fai; +}