Commit 8526ada2 authored by Anders Blomdell's avatar Anders Blomdell
Browse files

Added xattr handling

parent bc61a9db
......@@ -23,3 +23,6 @@ $(OUT_DIR)libhash.a(%.o): $(OUT_DIR)%.o
$(OUT_DIR)%.o: src/%.c Makefile
gcc -fPIC -Wall -Werror -c -o $@ $<
$(OUT_DIR)md5sum.o: ./src/libhash.h
$(OUT_DIR)sha512sum.o: ./src/libhash.h
use libc::{c_char, c_int, size_t};
use libc::{c_char, c_int, size_t, time_t};
use std::ffi::CString;
use std::os::unix::ffi::OsStrExt;
use std::path::Path;
use std::io::{Error, Result};
use std::os::unix::fs::{MetadataExt};
use std::fs::{Metadata};
#[link(name = "hash")]
extern {
fn hash_md5_file(path: *const c_char,
mtime: time_t,
mtime_nsec: i64,
flags: c_int,
maxage: time_t,
hash: &mut [u8;16]) -> c_int;
}
pub fn md5_file(path: &Path,
flags: c_int) -> Result<String>
metadata: Metadata,
flags: c_int,
maxage: time_t) -> Result<String>
{
let mut hash: [u8;16] = [0;16];
let cstring = CString::new(path.as_os_str().as_bytes()).unwrap();
if unsafe { hash_md5_file(cstring.as_ptr(), flags, &mut hash) } < 0 {
if unsafe { hash_md5_file(cstring.as_ptr(),
metadata.mtime(), metadata.mtime_nsec(), flags,
maxage, &mut hash) } < 0 {
Err(Error::last_os_error())
} else {
let s: Vec<String> = hash.iter().map(
......@@ -29,12 +38,10 @@ pub fn md5_file(path: &Path,
extern {
fn hash_md5_buf(buf: *const c_char,
length: size_t,
flags: c_int,
hash: &mut [u8;16]) -> c_int;
}
pub fn md5_symlink(path: &Path,
flags: c_int) -> Result<String>
pub fn md5_symlink(path: &Path) -> Result<String>
{
match path.read_link() {
Ok(symlink) => {
......@@ -42,7 +49,7 @@ pub fn md5_symlink(path: &Path,
let length = cstring.to_bytes().len();
let ptr = cstring.as_ptr();
let mut hash: [u8;16] = [0;16];
if unsafe { hash_md5_buf(ptr, length, flags, &mut hash) } < 0 {
if unsafe { hash_md5_buf(ptr, length, &mut hash) } < 0 {
Err(Error::last_os_error())
} else {
let s: Vec<String> = hash.iter().map(
......@@ -57,16 +64,23 @@ pub fn md5_symlink(path: &Path,
#[link(name = "hash")]
extern {
fn hash_sha512_file(path: *const c_char,
mtime: time_t,
mtime_nsec: i64,
flags: c_int,
maxage: time_t,
hash: &mut [u8;64]) -> c_int;
}
pub fn sha512_file(path: &Path,
flags: c_int) -> Result<String>
metadata: Metadata,
flags: c_int,
maxage: time_t) -> Result<String>
{
let mut hash: [u8;64] = [0;64];
let cstring = CString::new(path.as_os_str().as_bytes()).unwrap();
if unsafe { hash_sha512_file(cstring.as_ptr(), flags, &mut hash) } < 0 {
if unsafe { hash_sha512_file(cstring.as_ptr(),
metadata.mtime(), metadata.mtime_nsec(), flags,
maxage, &mut hash) } < 0 {
Err(Error::last_os_error())
} else {
let s: Vec<String> = hash.iter().map(
......@@ -79,12 +93,10 @@ pub fn sha512_file(path: &Path,
extern {
fn hash_sha512_buf(buf: *const c_char,
length: size_t,
flags: c_int,
hash: &mut [u8;64]) -> c_int;
}
pub fn sha512_symlink(path: &Path,
flags: c_int) -> Result<String>
pub fn sha512_symlink(path: &Path) -> Result<String>
{
match path.read_link() {
Ok(symlink) => {
......@@ -92,7 +104,7 @@ pub fn sha512_symlink(path: &Path,
let length = cstring.to_bytes().len();
let ptr = cstring.as_ptr();
let mut hash: [u8;64] = [0;64];
if unsafe { hash_sha512_buf(ptr, length, flags, &mut hash) } < 0 {
if unsafe { hash_sha512_buf(ptr, length, &mut hash) } < 0 {
Err(Error::last_os_error())
} else {
let s: Vec<String> = hash.iter().map(
......
......@@ -9,12 +9,13 @@ fi
## FLAGS_WRITE_XATTR std::os::raw::c_int 0x0002
## FLAGS_CLEAR_XATTR std::os::raw::c_int 0x0004
## FLAGS_MAX_AGE std::os::raw::c_int 0x0008
## FLAGS_NULL_TERMINATED std::os::raw::c_int 0x0010
## FLAGS_QUIET std::os::raw::c_int 0x0000
## FLAGS_NO_CALC_HASH std::os::raw::c_int 0x0010
## FLAGS_VERBOSE_MASK std::os::raw::c_int 0xc000
## FLAGS_VERBOSE0 std::os::raw::c_int 0x0000
## FLAGS_VERBOSE1 std::os::raw::c_int 0x0020
## FLAGS_VERBOSE2 std::os::raw::c_int 0x0040
## FLAGS_VERBOSE3 std::os::raw::c_int 0x0060
## FLAGS_VERBOSE1 std::os::raw::c_int 0x4000
## FLAGS_VERBOSE2 std::os::raw::c_int 0x8000
## FLAGS_VERBOSE3 std::os::raw::c_int 0xc000
gen_h() {
echo "enum {"
......
......@@ -6,19 +6,22 @@ use clap::{Arg, App};
use std::os::unix::fs::{FileTypeExt, MetadataExt};
use std::fs::{Metadata};
use std::path::{Path, PathBuf};
use std::sync::mpsc::{channel, Receiver};
use std::sync::mpsc::{channel, Receiver, Sender};
use std::thread;
use std::fmt;
use std::process::exit;
use threadpool::ThreadPool;
use std::collections::HashMap;
use std::cell::RefCell;
use libc::c_int;
use libc::{c_int, time_t};
use std::result::Result;
use std::io::Error as IOError;
use std::io::{self, Write};
mod libhash;
use libhash::{FLAGS_CLEAR_XATTR, FLAGS_READ_XATTR, FLAGS_WRITE_XATTR,
FLAGS_MAX_AGE, FLAGS_NO_CALC_HASH,
FLAGS_VERBOSE0, FLAGS_VERBOSE1, FLAGS_VERBOSE2, FLAGS_VERBOSE3};
mod hash;
use hash::{md5_file, md5_symlink, sha512_file, sha512_symlink};
mod walk;
......@@ -28,6 +31,25 @@ macro_rules! reportln {
($($arg:tt)*) => (writeln!(io::stderr(), $($arg)*).unwrap_or(()));
}
// Extract clap optin as an Option<t>
macro_rules! option_t_or_exit {
($m:ident, $v:expr, $t:ty) => (
if let Some(v) = $m.value_of($v) {
match v.trim().parse::<$t>() {
Ok(val) => Option::Some(val),
Err(_) => {
println!("Argument '--{} {}' cant be parsed as '{}'",
$v, v, stringify!($t));
exit(1);
}
}
} else {
Option::None
}
)
}
#[derive(Clone)]
struct Inode {
path: PathBuf,
metadata: Option<Metadata>
......@@ -39,8 +61,8 @@ impl std::fmt::Debug for Inode {
}}
enum WalkerMessage {
Start{path: PathBuf},
Inode{path: PathBuf, metadata: Metadata},
Start(PathBuf),
Inode(Inode), //path: PathBuf, metadata: Metadata},
Done
}
......@@ -67,17 +89,17 @@ enum HashOption<T, E> {
#[derive(Debug)]
struct HashEntry {
inode : Inode,
kind : char,
md5 : RefCell<HashOption<String, std::io::Error>>,
sha512 : RefCell<HashOption<String, std::io::Error>>
inode: Inode,
kind: char,
md5: RefCell<HashOption<String, std::io::Error>>,
sha512: RefCell<HashOption<String, std::io::Error>>
}
#[derive(Debug)]
struct WorkList {
first : usize,
last : usize,
list : HashMap<usize, HashEntry>
first: usize,
last: usize,
list: HashMap<usize, HashEntry>
}
#[allow(dead_code)]
......@@ -118,28 +140,97 @@ impl WorkList {
}
}
fn options_to_flags(options: &clap::ArgMatches) -> c_int {
struct WorkerPool<T> {
pool: ThreadPool,
pending: usize,
rx: Receiver<T>,
tx: Sender<T>
}
/*trait ExecuteIf {
fn execute_if<F, G, H>(&self, guard: bool, func: F)
where F: FnOnce() -> G + Send + 'static,
G: FnOnce() -> H + Send + 'static,
H: std::fmt::Debug;
}
impl ExecuteIf for WorkerPool {
*/
impl<T> WorkerPool<T> {
fn new(jobs: usize) -> WorkerPool<T> {
let (tx, rx) = channel::<T>();
WorkerPool {
pool: ThreadPool::new(jobs),
pending: 0,
rx: rx,
tx: tx
}
}
fn free(&self) -> bool {
self.pool.max_count() > self.pending
}
fn execute_if<F>(&mut self, guard: bool, func: F)
where F: FnOnce() -> T + Send + 'static,
T: Send + std::fmt::Debug + 'static {
if guard {
self.pending += 1;
let tx = self.tx.clone();
self.pool.execute(move || {
tx.send(func()).unwrap();
})
}
}
fn recv(&mut self) -> T {
self.pending -= 1;
self.rx.recv().unwrap()
}
}
fn options_to_flags(options: &clap::ArgMatches) -> (c_int, time_t) {
let xattr = options.is_present("xattr");
let read_xattr = options.is_present("read_xattr");
let write_xattr = options.is_present("write_xattr");
let clear_xattr = options.is_present("clear_xattr");
let mut flags : c_int = 0;
match (xattr, read_xattr, write_xattr, clear_xattr) {
(false, false, false, false) => {
flags = 0;
let maxage = option_t_or_exit!(options, "max_age", libc::time_t);
let mut flags = 0;
let mut age = 0;
match (xattr, read_xattr, write_xattr, clear_xattr, maxage) {
(false, false, false, false, None) => {
},
(true, false, false, false, None) |
(false, true, true, false, None) => {
flags = FLAGS_READ_XATTR | FLAGS_WRITE_XATTR;
},
(true, false, false, false, Some(maxage)) |
(false, true, true, false, Some(maxage)) => {
flags = FLAGS_READ_XATTR | FLAGS_WRITE_XATTR | FLAGS_MAX_AGE;
age = maxage;
},
(false, true, false, false, None) => {
flags = FLAGS_READ_XATTR;
},
(true, false, false, false) |
(false, true, true, false) => {
flags |= libhash::FLAGS_READ_XATTR | libhash::FLAGS_WRITE_XATTR;
(false, true, false, false, Some(maxage)) => {
flags = FLAGS_READ_XATTR | FLAGS_MAX_AGE;
age = maxage;
},
(false, false, false, true) => {
flags |= libhash::FLAGS_CLEAR_XATTR;
(false, false, false, true, None) => {
flags = FLAGS_CLEAR_XATTR | FLAGS_NO_CALC_HASH;
},
(_, _, _, _) => {
(_, _, _, _, Some(age)) => {
panic!("--maxage={} specified without sane xattr flags", age);
},
(_, _, _, _, _) => {
panic!("Invalid flags");
}
}
flags
match (options.is_present("quiet"), options.occurrences_of("verbose")) {
(true, 0) => flags |= FLAGS_VERBOSE0,
(false, 0) => flags |= FLAGS_VERBOSE1,
(false, 1) => flags |= FLAGS_VERBOSE2,
(false, 2) => flags |= FLAGS_VERBOSE3,
(_, _) => panic!("Invalid verbosity")
}
(flags, age)
}
fn file_type_to_flag(file_type: &std::fs::FileType) -> char {
......@@ -155,31 +246,14 @@ fn file_type_to_flag(file_type: &std::fs::FileType) -> char {
fn dispatcher(options: clap::ArgMatches,
from_walker: Receiver<WalkerMessage>) {
macro_rules! parse_opt {
($n:expr, $t:ty) => (
if let Some(v) = options.value_of($n) {
match v.parse::<$t>() {
Ok(val) => Option::Some(val),
Err(_) => {
println!("Argument '--{} {}' cant be parsed as '{}'",
$n, v, stringify!($t));
exit(1);
}
}
} else {
Option::None
}
)
};
let jobs = parse_opt!("jobs", usize).unwrap_or(1);
let lookahead = parse_opt!("lookahead", usize).unwrap_or(jobs * 10);
let jobs = option_t_or_exit!(options, "jobs", usize).unwrap_or(1);
let lookahead = option_t_or_exit!(options, "lookahead", usize)
.unwrap_or(jobs * 10);
let calc_md5 = options.is_present("md5");
let calc_sha512 = options.is_present("sha512");
let flags = options_to_flags(&options);
let mut pending = 0 as usize;
let pool = ThreadPool::new(jobs);
let (flags, maxage) = options_to_flags(&options);
let mut pool = WorkerPool::<WorkerMessage>::new(jobs);
let mut worklist = WorkList::new();
let (worker_to_dispatcher, from_pool) = channel::<WorkerMessage>();
let mut done = false;
print!("#fields: size:mtime:uid:gid:mode:");
if calc_md5 { print!("md5:"); }
......@@ -189,14 +263,14 @@ fn dispatcher(options: clap::ArgMatches,
// reportln!("XXX {} {} {}", done, pending, worklist.used());
if done && worklist.used() == 0 {
break;
} if !done && pending < jobs && worklist.used() < lookahead {
// Get paths to checksum
} if !done && pool.free() && worklist.used() < lookahead {
// Get paths from walker and enqueue work
let message = match from_walker.recv() {
Ok(message) => message,
Err(_) => break
};
match message {
WalkerMessage::Start{path} => {
WalkerMessage::Start(path) => {
worklist.push_back(HashEntry {
inode: Inode {
path:path.clone(),
......@@ -207,8 +281,10 @@ fn dispatcher(options: clap::ArgMatches,
sha512: RefCell::new(HashOption::Unused),
});
},
WalkerMessage::Inode{path, metadata} => {
WalkerMessage::Inode(inode) => {
// Push to worklist
let path = inode.path;
let metadata = inode.metadata.unwrap();
use HashOption::{Pending, Unused, None};
let (kind, md5, sha512) = match
file_type_to_flag(&metadata.file_type()) {
......@@ -233,34 +309,58 @@ fn dispatcher(options: clap::ArgMatches,
sha512: RefCell::new(sha512),
});
// Spawn off needed work
macro_rules! execute {
($guard:expr, $func:expr, $message:ident) => (
if $guard {
let path = path.clone();
let tx = worker_to_dispatcher.clone();
let flags = flags;
pending += 1;
pool.execute(move || {
let h = $func(path.as_path(), flags);
tx.send( WorkerMessage {
index: index,
hash:$message(h)
}).unwrap();
});
}
)
};
match kind {
'F' => {
use WorkerMessageHash::{MD5, SHA512};
execute!(calc_md5, md5_file, MD5);
execute!(calc_sha512, sha512_file, SHA512);
pool.execute_if(calc_md5, {
let path = path.clone();
let metadata = metadata.clone();
move || {
let h = md5_file(path.as_path(),
metadata,
flags, maxage);
WorkerMessage {
index: index,
hash: MD5(h)
}
}
});
pool.execute_if(calc_sha512, {
let path = path.clone();
move || {
let h = sha512_file(path.as_path(),
metadata,
flags, maxage);
WorkerMessage {
index: index,
hash: SHA512(h)
}
}
});
},
'L' => {
use WorkerMessageHash::{MD5, SHA512};
execute!(calc_md5, md5_symlink, MD5);
execute!(calc_sha512, sha512_symlink, SHA512);
pool.execute_if(calc_md5, {
let path = path.clone();
move || {
let h = md5_symlink(path.as_path());
WorkerMessage {
index: index,
hash: MD5(h)
}
}
});
pool.execute_if(calc_sha512, {
let path = path.clone();
move || {
let h = sha512_symlink(path.as_path());
WorkerMessage {
index: index,
hash: SHA512(h)
}
}
});
},
_ => ()
};
......@@ -268,12 +368,11 @@ fn dispatcher(options: clap::ArgMatches,
WalkerMessage::Done =>
done = true,
}
} else if pending > 0 {
// Handle paths being processed
let message = from_pool.recv().unwrap();
} else if pool.pending > 0 {
// Handle finished results
let message = pool.recv();
let index = message.index;
let item = worklist.get(index).unwrap();
pending -= 1;
use WorkerMessageHash::{MD5,SHA512};
match message.hash {
MD5(hash) => {
......@@ -312,8 +411,11 @@ fn dispatcher(options: clap::ArgMatches,
let error = match ( md5, sha512 ) {
(&Err(ref e @ _), _) => Option::Some(e),
(_, &Err(ref e @ _)) => Option::Some(e),
(_,_) => Option::None
(_, _) => Option::None
};
if flags & FLAGS_CLEAR_XATTR != 0 {
continue;
}
match error {
Option::Some(e) => {
// Failed to checksum file, report and drop
......@@ -412,8 +514,13 @@ fn main() {
.help("Calculate SHA512 hash"))
.arg(Arg::with_name("verbose")
.short("v")
.conflicts_with("quiet")
.multiple(true)
.help("Sets the level of verbosity"))
.arg(Arg::with_name("quiet")
.short("q")
.conflicts_with("verbose")
.help("Sets the level of verbosity"))
.arg(Arg::with_name("max_age")
.short("m")
.long("max-age")
......@@ -434,15 +541,15 @@ fn main() {
(worker, tx)
};
let callback = | p:&Path, m:&Metadata | {
tx.send( WalkerMessage::Inode{
path:p.to_owned(), metadata:m.clone()}).unwrap();
tx.send( WalkerMessage::Inode(
Inode{path:p.to_owned(), metadata:Some(m.clone())})).unwrap();
};
let paths : Vec<_> = matches.values_of_os("PATH").unwrap().collect();
// let mut paths : Vec<_> = matches.values_of_os("PATH").unwrap().collect();
// paths.sort();
for p in paths {
let path = std::path::Path::new(p);
let _ = tx.send(WalkerMessage::Start{path: path.to_owned()});
let _ = tx.send(WalkerMessage::Start(path.to_owned()));
walk::visit(std::path::Path::new(p), &callback).unwrap();
}
tx.send(WalkerMessage::Done).unwrap();
......
......@@ -2,21 +2,31 @@
#include <string.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/xattr.h>
#include <fcntl.h>
#include <time.h>
#include <unistd.h>
#include <openssl/md5.h>
#include <errno.h>
#include <limits.h>
#include <stdint.h>
#include "libhash.h"
#define BLOCKSIZE 4096
#define XATTR_MD5HASH "trusted.md5.control.lth.se"
struct __attribute__((__packed__)) hash_attr {
time_t hash_time;
time_t mtime;
uint32_t mtime_nsec;
uint8_t hash[MD5_DIGEST_LENGTH];
};
int hash_md5_buf(char *buf,
size_t length,
int flags,
unsigned char *hash)
{
MD5_CTX ctx;
MD5_Init(&ctx);
......@@ -27,14 +37,66 @@ int hash_md5_buf(char *buf,
}
int hash_md5_file(char *filename,
time_t mtime,
int64_t mtime_nsec,
int flags,
time_t maxage,
unsigned char *hash)
{
int fd;
int result = 0, recalc = 1;
struct hash_attr hash_attr;
if (flags & FLAGS_CLEAR_XATTR) {
if (removexattr(filename, XATTR_MD5HASH) != 0) {
if (errno != ENODATA) {
result = -1;
fprintf(stderr, "Failed to remove xattr '%s'\n", filename);
goto out;
} else if ((flags & FLAGS_VERBOSE_MASK) > FLAGS_VERBOSE2) {
fprintf(stderr, "No xattr for '%s'\n", filename);
}
} else {
if ((flags & FLAGS_VERBOSE_MASK) > FLAGS_VERBOSE2) {
fprintf(stderr, "Removed xattr for '%s'\n", filename);
}
}
}
if (flags & FLAGS_NO_CALC_HASH) {
// We should not calculate hash
goto out;
}
if (flags & FLAGS_READ_XATTR) {
int err;
err = getxattr(filename, XATTR_MD5HASH, &hash_attr, sizeof(hash_attr));
if (err < 0 && errno != ENODATA) {
result = -1;
goto out;
}
if (err != sizeof(hash_attr)) {
recalc = 1;
} else if (le64toh(hash_attr.mtime) != mtime) {
recalc = 1;
} else if (le32toh(hash_attr.mtime_nsec) != mtime_nsec) {
recalc = 1;
} else if (flags & FLAGS_MAX_AGE && le64toh(hash_attr.hash_time) < maxage) {
recalc = 1;
} else {