Added a USN carver (#94)

bin/carve.go

@@ -0,0 +1,66 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	kingpin "gopkg.in/alecthomas/kingpin.v2"
+	"www.velocidex.com/golang/go-ntfs/parser"
+)
+
+var (
+	carve_command = app.Command(
+		"carve", "Carve USN records from the disk.")
+
+	carve_command_file_arg = carve_command.Arg(
+		"file", "The image file to inspect",
+	).Required().File()
+)
+
+const carve_template = `
+USN ID: %#x @ %d
+Filename: %s
+FullPath: %s
+Timestamp: %v
+Reason: %s
+FileAttributes: %s
+SourceInfo: %s
+`
+
+func doCarve() {
+	reader, _ := parser.NewPagedReader(
+		getReader(*carve_command_file_arg), 1024, 10000)
+
+	ntfs_ctx, err := parser.GetNTFSContext(reader, 0)
+	kingpin.FatalIfError(err, "Can not open filesystem")
+
+	size := ntfs_ctx.Boot.VolumeSize() * int64(ntfs_ctx.Boot.Sector_size())
+	fmt.Printf("VolumeSize %v\n", size)
+
+	for record := range parser.CarveUSN(
+		context.Background(), ntfs_ctx, reader, size) {
+
+		filename := record.Filename()
+
+		fmt.Printf(carve_template, record.Usn(), record.DiskOffset,
+			filename,
+			record.Links(), record.TimeStamp(),
+			strings.Join(record.Reason(), ", "),
+			strings.Join(record.FileAttributes(), ", "),
+			strings.Join(record.SourceInfo(), ", "),
+		)
+	}
+}
+
+func init() {
+	command_handlers = append(command_handlers, func(command string) bool {
+		switch command {
+		case carve_command.FullCommand():
+			doCarve()
+		default:
+			return false
+		}
+		return true
+	})
+}

parser/boot.go

@@ -20,6 +20,10 @@ func (self *NTFS_BOOT_SECTOR) BlockCount() int64 {
 	return int64(self._volume_size()) / int64(self.ClusterSize())
 }
 
+func (self *NTFS_BOOT_SECTOR) VolumeSize() int64 {
+	return int64(self._volume_size())
+}
+
 func (self *NTFS_BOOT_SECTOR) RecordSize() int64 {
 	_record_size := int64(self._mft_record_size())
 	if _record_size > 0 {

parser/context.go

@@ -47,7 +47,7 @@ func newNTFSContext(image io.ReaderAt, name string) *NTFSContext {
 	ntfs.mft_summary_cache = NewMFTEntryCache(ntfs)
 	ntfs.full_path_resolver = &FullPathResolver{
 		ntfs:      ntfs,
-		options:   ntfs.options,
+		options:   &ntfs.options,
 		mft_cache: ntfs.mft_summary_cache,
 	}
 

parser/hardlinks.go

@@ -45,11 +45,10 @@ func (self *Visitor) Components() [][]string {
 	result := make([][]string, 0, len(self.Paths))
 
 	for _, p := range self.Paths {
+		p = append(p, self.Prefix...)
 		ReverseStringSlice(p)
-		components := append([]string{}, self.Prefix...)
-		components = append(components, p...)
 		if len(p) > 0 {
-			result = append(result, components)
+			result = append(result, p)
 		}
 	}
 	return result
@@ -61,7 +60,7 @@ func (self *Visitor) Components() [][]string {
 // MFT to reconstruct the full path of an mft entry.
 type FullPathResolver struct {
 	ntfs    *NTFSContext
-	options Options
+	options *Options
 
 	mft_cache *MFTEntryCache
 }

parser/reader.go

@@ -67,6 +67,12 @@ func (self *PagedReader) ReadAt(buf []byte, offset int64) (int, error) {
 	self.mu.Lock()
 	defer self.mu.Unlock()
 
+	// If the read is very large and a multiple of pagesize, it is
+	// faster to just delegate reading to the underlying reader.
+	if len(buf) > 10*int(self.pagesize) && len(buf)%int(self.pagesize) == 0 {
+		return self.reader.ReadAt(buf, offset)
+	}
+
 	buf_idx := 0
 	for {
 		// How much is left in this page to read?

parser/usn.go

@@ -1,6 +1,7 @@
 package parser
 
 import (
+	"bytes"
 	"context"
 	"errors"
 	"fmt"
@@ -346,3 +347,116 @@ func WatchUSN(ctx context.Context, ntfs_ctx *NTFSContext, period int) chan *USN_
 
 	return output
 }
+
+type USNCarvedRecord struct {
+	*USN_RECORD
+	DiskOffset int64
+}
+
+func CarveUSN(ctx context.Context,
+	ntfs_ctx *NTFSContext,
+	stream io.ReaderAt,
+	size int64) chan *USNCarvedRecord {
+	output := make(chan *USNCarvedRecord)
+
+	go func() {
+		defer close(output)
+
+		cluster_size := ntfs_ctx.ClusterSize
+		if cluster_size == 0 {
+			cluster_size = 0x1000
+		}
+
+		buffer_size := 1024 * cluster_size
+
+		buffer := make([]byte, buffer_size)
+
+		now := time.Now()
+
+		// Overlap buffers in case an entry is split
+		for i := int64(0); i < size; i += buffer_size - cluster_size {
+			select {
+			case <-ctx.Done():
+				return
+			default:
+			}
+
+			DebugPrint("%v: Reading buffer length %v at %v in %v\n",
+				time.Now(), len(buffer), i, time.Now().Sub(now))
+
+			now = time.Now()
+			n, err := stream.ReadAt(buffer, i)
+			if err != nil && err != io.EOF {
+				return
+			}
+
+			if n < 64 {
+				return
+			}
+
+			buf_reader := bytes.NewReader(buffer[:n])
+
+			// We assume that entries are aligned to 0x10 at least.
+			for j := int64(0); j < int64(n)-0x10; j += 0x10 {
+
+				// MajorVersion must be 2 and MinorVersion 0. This is
+				// a quick check that should eliminate most of the
+				// false positives. We check more carefully below.
+				if buffer[j+4] != '\x02' ||
+					buffer[j+5] != '\x00' ||
+					buffer[j+6] != '\x00' ||
+					buffer[j+7] != '\x00' {
+					continue
+				}
+
+				record, ok := testUSNEntry(ntfs_ctx, buf_reader, j)
+				if !ok {
+					continue
+				}
+
+				select {
+				case <-ctx.Done():
+					return
+
+				case output <- &USNCarvedRecord{
+					USN_RECORD: record,
+					DiskOffset: j + i,
+				}:
+				}
+			}
+		}
+	}()
+
+	return output
+}
+
+var (
+	year2020 = time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)
+	year2040 = time.Date(2040, 1, 1, 0, 0, 0, 0, time.UTC)
+)
+
+func testUSNEntry(ntfs_ctx *NTFSContext,
+	reader io.ReaderAt, offset int64) (*USN_RECORD, bool) {
+
+	record := ntfs_ctx.Profile.USN_RECORD_V2(reader, offset)
+	record_length := record.RecordLength()
+	if record_length < 64 || record_length > 1024 {
+		return nil, false
+	}
+
+	if record.FileNameOffset() > 255 ||
+		record.FileNameLength() > 255 {
+		return nil, false
+	}
+
+	// Check the the time is reasonable
+	ts := record.TimeStamp()
+	if ts.Before(year2020) || ts.After(year2040) {
+		return nil, false
+	}
+
+	return &USN_RECORD{
+		USN_RECORD_V2: record,
+		context:       ntfs_ctx,
+	}, true
+}