完善 Windows 运维封装与 NTFS 索引解析

- 新增自启动幂等配置、统一错误语义、进程等待和进程树终止能力
- 增强服务生命周期管理,支持等待状态、重启、幂等创建和配置更新
- 新增 NTFS 卷索引、文件 ID 解析、文件遍历、USN 变更监听和 bookmark 持久化
- 修复 NTFS boot sector、fragment、MFT、USN 解析边界和路径重建问题
- 补充权限、进程、服务、NTFS 解析和工作流回归测试
- 增加 Windows 测试脚本和管理员 NTFS smoke 验证脚本
- 升级 Go 兼容版本到 1.18,并更新 stario、win32api 及相关间接依赖
This commit is contained in:
2026-06-09 15:59:31 +08:00
parent feb1a21da8
commit 7e6cc73106
31 changed files with 4937 additions and 981 deletions
+3
View File
@@ -50,6 +50,9 @@ func newFileStatFromInformation(d *syscall.ByHandleFileInformation, name string,
LastWriteTime: d.LastWriteTime,
FileSizeHigh: d.FileSizeHigh,
FileSizeLow: d.FileSizeLow,
vol: d.VolumeSerialNumber,
idxhi: d.FileIndexHigh,
idxlo: d.FileIndexLow,
}
}
+393 -6
View File
@@ -1,13 +1,400 @@
package usn
import (
"fmt"
"encoding/binary"
"errors"
"os"
"path/filepath"
"strings"
"syscall"
"testing"
"unicode/utf16"
"b612.me/win32api"
)
func Test_USN(t *testing.T) {
fmt.Println("start")
data, err := ListUsnFile("C:\\")
fmt.Println(err)
fmt.Println(len(data))
func TestGetPointerUsesSliceLength(t *testing.T) {
buf := make([]uint16, 3, 16)
_, size, err := getPointer(buf)
if err != nil {
t.Fatalf("getPointer failed: %v", err)
}
if want := uintptr(len(buf)) * uintptr(2); size != want {
t.Fatalf("slice size = %d, want %d", size, want)
}
}
func TestParseUSNOutput(t *testing.T) {
buf := buildTestUSNBuffer(1234, "hello.txt", false, 0x20)
var got usnRecordData
next, err := parseUSNOutput(buf, uint32(len(buf)), func(record usnRecordData) error {
got = record
return nil
})
if err != nil {
t.Fatalf("parseUSNOutput failed: %v", err)
}
if next != 1234 {
t.Fatalf("next = %d, want 1234", next)
}
if got.FileName != "hello.txt" {
t.Fatalf("FileName = %q, want %q", got.FileName, "hello.txt")
}
if got.FileReferenceNumber != 100 {
t.Fatalf("FileReferenceNumber = %d, want 100", got.FileReferenceNumber)
}
if got.ParentFileReferenceNumber != 55 {
t.Fatalf("ParentFileReferenceNumber = %d, want 55", got.ParentFileReferenceNumber)
}
if got.Reason != 0x20 {
t.Fatalf("Reason = %#x, want %#x", got.Reason, 0x20)
}
}
func TestParseUSNOutputRejectsShortRecord(t *testing.T) {
buf := buildTestUSNBuffer(1, "bad", false, 0)
binary.LittleEndian.PutUint32(buf[usnBufferHeaderSize:], uint32(usnRecordMinSize-2))
if _, err := parseUSNOutput(buf, uint32(len(buf)), func(usnRecordData) error { return nil }); err == nil {
t.Fatal("expected parseUSNOutput to reject short record")
}
}
func TestShouldPreferUSNFileName(t *testing.T) {
tests := []struct {
current string
candidate string
want bool
}{
{current: "", candidate: "Program Files", want: true},
{current: "PROGRA~1", candidate: "Program Files", want: true},
{current: "Program Files", candidate: "PROGRA~1", want: false},
{current: "abc", candidate: "abcdef", want: true},
{current: "abcdef", candidate: "abc", want: false},
{current: "Program Files", candidate: "program files", want: false},
}
for _, tt := range tests {
if got := shouldPreferUSNFileName(tt.current, tt.candidate); got != tt.want {
t.Fatalf("shouldPreferUSNFileName(%q, %q) = %v, want %v", tt.current, tt.candidate, got, tt.want)
}
}
}
func TestMergeUSNFileEntryPrefersLongName(t *testing.T) {
current := FileEntry{Name: "PROGRA~1", Parent: 7}
candidate := FileEntry{Name: "Program Files", Parent: 9}
merged := mergeUSNFileEntry(current, candidate)
if merged.Name != "Program Files" {
t.Fatalf("Name = %q, want %q", merged.Name, "Program Files")
}
if merged.Parent != 9 {
t.Fatalf("Parent = %d, want 9", merged.Parent)
}
}
func TestMergeUSNFileEntryTracksRename(t *testing.T) {
current := FileEntry{Name: "alpha.txt", Parent: 7}
candidate := FileEntry{Name: "omega.txt", Parent: 7}
merged := mergeUSNFileEntry(current, candidate)
if merged.Name != "omega.txt" {
t.Fatalf("Name = %q, want %q", merged.Name, "omega.txt")
}
}
func TestFilterUSNFileMapUsesFinalName(t *testing.T) {
fileMap := map[win32api.DWORDLONG]FileEntry{
1: {Name: "Windows", Parent: 1, Type: 1},
2: {Name: "Program Files", Parent: 1, Type: 0},
3: {Name: "Temp", Parent: 1, Type: 0},
}
filtered := filterUSNFileMap(fileMap, func(name string, _ bool) bool {
return strings.Contains(name, "Program")
})
if _, ok := filtered[1]; !ok {
t.Fatal("expected directory entry to be retained")
}
if _, ok := filtered[2]; !ok {
t.Fatal("expected matching file entry to be retained")
}
if _, ok := filtered[3]; ok {
t.Fatal("did not expect non-matching file entry to be retained")
}
}
func TestNeedPathCanonicalNameOverlay(t *testing.T) {
if needPathCanonicalNameOverlay(map[win32api.DWORDLONG]FileEntry{
1: {Name: "Program Files", Parent: 1},
}) {
t.Fatal("did not expect overlay for long names only")
}
if !needPathCanonicalNameOverlay(map[win32api.DWORDLONG]FileEntry{
1: {Name: "PROGRA~1", Parent: 1},
}) {
t.Fatal("expected overlay when short name exists")
}
}
func TestWindowsBaseName(t *testing.T) {
if got := windowsBaseName(`C:\Program Files\`); got != "Program Files" {
t.Fatalf("windowsBaseName returned %q", got)
}
if got := windowsBaseName(`C:\Windows\System32`); got != "System32" {
t.Fatalf("windowsBaseName returned %q", got)
}
if got := windowsBaseName(`single`); got != "single" {
t.Fatalf("windowsBaseName returned %q", got)
}
}
func TestApplyPathCanonicalNamesUsesNormalizedPath(t *testing.T) {
origNormalize := normalizePathForUSN
defer func() {
normalizePathForUSN = origNormalize
}()
normalizePathForUSN = func(path string) string {
if strings.Contains(path, "PROGRA~1") {
return strings.Replace(path, "PROGRA~1", "Program Files", 1)
}
return path
}
fileMap := map[win32api.DWORDLONG]FileEntry{
1: {Name: "", Parent: 1, Type: 1},
2: {Name: "PROGRA~1", Parent: 1, Type: 0},
}
applyPathCanonicalNames("C:\\", fileMap)
entry := fileMap[2]
if entry.Name != "Program Files" {
t.Fatalf("Name = %q, want %q", entry.Name, "Program Files")
}
if entry.Parent != 1 {
t.Fatalf("Parent = %d, want 1", entry.Parent)
}
}
func TestApplyPathCanonicalNamesSkipsWhenNotNeeded(t *testing.T) {
origNormalize := normalizePathForUSN
defer func() {
normalizePathForUSN = origNormalize
}()
called := false
normalizePathForUSN = func(path string) string {
called = true
return path
}
fileMap := map[win32api.DWORDLONG]FileEntry{
2: {Name: "Program Files", Parent: 1, Type: 0},
}
applyPathCanonicalNames("C:\\", fileMap)
if called {
t.Fatal("did not expect normalization when no short names exist")
}
}
func TestFileStatFromIDWithfd(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "usn-by-id.txt")
content := []byte("usn by id test")
if err := os.WriteFile(path, content, 0600); err != nil {
t.Fatalf("WriteFile failed: %v", err)
}
volume := filepath.VolumeName(path) + `\`
info, err := GetDiskInfo(volume)
if err != nil {
t.Fatalf("GetDiskInfo failed: %v", err)
}
if !strings.EqualFold(info.Format, "NTFS") {
t.Skipf("volume %s is %s, not NTFS", volume, info.Format)
}
file, err := os.Open(path)
if err != nil {
t.Fatalf("Open failed: %v", err)
}
defer file.Close()
var handleInfo syscall.ByHandleFileInformation
if err := syscall.GetFileInformationByHandle(syscall.Handle(file.Fd()), &handleInfo); err != nil {
t.Fatalf("GetFileInformationByHandle failed: %v", err)
}
volumeHandle, err := CreateFile(`\\.\`+volume[:len(volume)-1], syscall.O_RDONLY, win32api.FILE_ATTRIBUTE_NORMAL)
if err != nil {
if errors.Is(err, syscall.ERROR_ACCESS_DENIED) {
t.Skipf("opening volume handle requires extra privilege: %v", err)
}
t.Fatalf("CreateFile(volume) failed: %v", err)
}
defer syscall.Close(volumeHandle)
fileID := win32api.DWORDLONG(uint64(handleInfo.FileIndexHigh)<<32 | uint64(handleInfo.FileIndexLow))
stat, err := fileStatFromIDWithfd(volumeHandle, fileID, filepath.Base(path), path, 0)
if err != nil {
t.Fatalf("fileStatFromIDWithfd failed: %v", err)
}
if stat.Name() != filepath.Base(path) {
t.Fatalf("Name = %q, want %q", stat.Name(), filepath.Base(path))
}
if stat.Size() != int64(len(content)) {
t.Fatalf("Size = %d, want %d", stat.Size(), len(content))
}
if stat.vol != handleInfo.VolumeSerialNumber || stat.idxhi != handleInfo.FileIndexHigh || stat.idxlo != handleInfo.FileIndexLow {
t.Fatal("file identifiers do not match source handle info")
}
}
func TestCollectUSNFileStatsSkipsFailedFetch(t *testing.T) {
data := map[win32api.DWORDLONG]FileEntry{
1: {Name: "keep-a.txt", Parent: 1, Type: 0},
2: {Name: "drop-b.txt", Parent: 1, Type: 0},
3: {Name: "keep-c", Parent: 1, Type: 1},
}
got := collectUSNFileStats(data, nil, func(id win32api.DWORDLONG, entry FileEntry) (FileStat, error) {
if id == 2 {
return FileStat{}, errors.New("fetch failed")
}
stat := FileStat{name: entry.Name}
if entry.Type == 1 {
stat.FileAttributes = win32api.FILE_ATTRIBUTE_DIRECTORY
}
return stat, nil
})
if len(got) != 2 {
t.Fatalf("len(got) = %d, want 2", len(got))
}
names := map[string]bool{}
for _, stat := range got {
names[stat.Name()] = true
if stat.Name() == "" {
t.Fatal("expected failed fetch entries to be skipped instead of zero-value placeholders")
}
}
if !names["keep-a.txt"] || !names["keep-c"] {
t.Fatalf("unexpected names: %+v", names)
}
if names["drop-b.txt"] {
t.Fatal("did not expect failed fetch entry in results")
}
}
func TestCollectUSNFileStatsAppliesFilter(t *testing.T) {
data := map[win32api.DWORDLONG]FileEntry{
1: {Name: "keep-file.txt", Parent: 1, Type: 0},
2: {Name: "skip-file.txt", Parent: 1, Type: 0},
3: {Name: "keep-dir", Parent: 1, Type: 1},
}
got := collectUSNFileStats(data, func(name string, _ bool) bool {
return strings.HasPrefix(name, "keep-")
}, func(_ win32api.DWORDLONG, entry FileEntry) (FileStat, error) {
stat := FileStat{name: entry.Name}
if entry.Type == 1 {
stat.FileAttributes = win32api.FILE_ATTRIBUTE_DIRECTORY
}
return stat, nil
})
if len(got) != 2 {
t.Fatalf("len(got) = %d, want 2", len(got))
}
for _, stat := range got {
if !strings.HasPrefix(stat.Name(), "keep-") {
t.Fatalf("unexpected stat name %q", stat.Name())
}
}
}
func TestCollectUSNFileStatsNilFilterIncludesAll(t *testing.T) {
data := map[win32api.DWORDLONG]FileEntry{
1: {Name: "a.txt", Parent: 1, Type: 0},
2: {Name: "b.txt", Parent: 1, Type: 0},
3: {Name: "c", Parent: 1, Type: 1},
}
got := collectUSNFileStats(data, nil, func(_ win32api.DWORDLONG, entry FileEntry) (FileStat, error) {
return FileStat{name: entry.Name}, nil
})
if len(got) != len(data) {
t.Fatalf("len(got) = %d, want %d", len(got), len(data))
}
}
func TestCollectUSNFileStatsNilFetchReturnsEmpty(t *testing.T) {
data := map[win32api.DWORDLONG]FileEntry{
1: {Name: "a.txt", Parent: 1, Type: 0},
2: {Name: "b.txt", Parent: 1, Type: 0},
}
got := collectUSNFileStats(data, nil, nil)
if len(got) != 0 {
t.Fatalf("len(got) = %d, want 0", len(got))
}
}
func buildTestUSNBuffer(next uint64, name string, isDir bool, reason uint32) []byte {
encoded := utf16.Encode([]rune(name))
nameBytes := make([]byte, len(encoded)*2)
for i, v := range encoded {
binary.LittleEndian.PutUint16(nameBytes[i*2:], v)
}
recordLength := usnRecordMinSize + len(nameBytes)
buf := make([]byte, usnBufferHeaderSize+recordLength)
binary.LittleEndian.PutUint64(buf[:usnBufferHeaderSize], next)
record := buf[usnBufferHeaderSize:]
binary.LittleEndian.PutUint32(record, uint32(recordLength))
binary.LittleEndian.PutUint16(record[4:], 2)
binary.LittleEndian.PutUint16(record[6:], 0)
binary.LittleEndian.PutUint64(record[usnRecordOffsetFileReference:], 100)
binary.LittleEndian.PutUint64(record[usnRecordOffsetParentReference:], 55)
binary.LittleEndian.PutUint32(record[usnRecordOffsetReason:], reason)
attrs := uint32(0)
if isDir {
attrs = win32api.FILE_ATTRIBUTE_DIRECTORY
}
binary.LittleEndian.PutUint32(record[usnRecordOffsetFileAttributes:], attrs)
binary.LittleEndian.PutUint16(record[usnRecordOffsetFileNameLength:], uint16(len(nameBytes)))
binary.LittleEndian.PutUint16(record[usnRecordOffsetFileNameOffset:], uint16(usnRecordMinSize))
copy(record[usnRecordMinSize:], nameBytes)
return buf
}
func TestNormalizeDiskName(t *testing.T) {
tests := map[string]string{
"c:": "C:\\",
"c:\\temp": "C:\\",
"D:/data": "D:\\",
}
for input, want := range tests {
got, err := normalizeDiskName(input)
if err != nil {
t.Fatalf("normalizeDiskName(%q) returned error: %v", input, err)
}
if got != want {
t.Fatalf("normalizeDiskName(%q) = %q, want %q", input, got, want)
}
}
if _, err := normalizeDiskName(""); err == nil {
t.Fatal("expected empty disk name error")
}
if _, err := normalizeDiskName("not-a-drive"); err == nil {
t.Fatal("expected invalid disk name error")
}
}
func TestUSNReasonStringUnknownHighBitDoesNotPanic(t *testing.T) {
got := USNReasonString(0x80000000)
if got == "" {
t.Fatal("expected non-empty reason string")
}
}
+492 -178
View File
@@ -3,10 +3,12 @@ package usn
import (
"b612.me/stario"
"b612.me/win32api"
"encoding/binary"
"fmt"
"os"
"path/filepath"
"reflect"
"runtime"
"strings"
"syscall"
"unsafe"
)
@@ -18,6 +20,29 @@ type DiskInfo struct {
SerialNumber uint32
}
func normalizeDiskName(diskName string) (string, error) {
name := strings.TrimSpace(strings.ReplaceAll(diskName, "/", "\\"))
if name == "" {
return "", fmt.Errorf("empty disk name")
}
volume := filepath.VolumeName(name)
if len(volume) == 2 && volume[1] == ':' {
return strings.ToUpper(volume[:1]) + ":\\", nil
}
if len(name) >= 2 && name[1] == ':' {
return strings.ToUpper(name[:1]) + ":\\", nil
}
return "", fmt.Errorf("invalid disk name: %q", diskName)
}
func volumeDevicePath(diskName string) (string, error) {
normalized, err := normalizeDiskName(diskName)
if err != nil {
return "", err
}
return "\\\\.\\" + strings.TrimSuffix(normalized, "\\"), nil
}
func ListDrivers() ([]string, error) {
drivers := make([]string, 0, 26)
buf := make([]uint16, 255)
@@ -70,27 +95,42 @@ func GetDiskInfo(disk string) (DiskInfo, error) {
}
func DeviceIoControl(handle syscall.Handle, controlCode uint32, in interface{}, out interface{}, done *uint32) (err error) {
inPtr, inSize := getPointer(in)
outPtr, outSize := getPointer(out)
inPtr, inSize, err := getPointer(in)
if err != nil {
return err
}
outPtr, outSize, err := getPointer(out)
if err != nil {
return err
}
//_,err = syscall.Syscall9(procDeviceIoControl.Addr(), 8, uintptr(handle), uintptr(controlCode), inPtr, uintptr(inSize), outPtr, uintptr(outSize), uintptr(unsafe.Pointer(done)), uintptr(0), 0)
_, err = win32api.DeviceIoControlPtr(win32api.HANDLE(handle), win32api.DWORD(controlCode), inPtr, win32api.DWORD(inSize), outPtr, win32api.DWORD(outSize), done, nil)
return
}
func getPointer(i interface{}) (pointer, size uintptr) {
func getPointer(i interface{}) (pointer uintptr, size uintptr, err error) {
if i == nil {
return 0, 0, nil
}
v := reflect.ValueOf(i)
switch k := v.Kind(); k {
case reflect.Ptr:
if v.IsNil() {
return 0, 0, nil
}
t := v.Elem().Type()
size = t.Size()
pointer = v.Pointer()
case reflect.Slice:
size = uintptr(v.Cap())
if v.Len() == 0 {
return 0, 0, nil
}
size = uintptr(v.Len()) * v.Type().Elem().Size()
pointer = v.Pointer()
default:
fmt.Println("error")
return 0, 0, fmt.Errorf("unsupported DeviceIoControl buffer type %T", i)
}
return
return pointer, size, nil
}
// Need a custom Open to work with backup_semantics
@@ -179,13 +219,209 @@ type FileMonitor struct {
Reason string
}
func ListUsnFile(driver string) (map[win32api.DWORDLONG]FileEntry, error) {
var normalizePathForUSN = normalizeExistingLongPath
const (
usnBufferHeaderSize = int(unsafe.Sizeof(win32api.USN(0)))
usnRecordMinSize = int(unsafe.Offsetof(win32api.USN_RECORD{}.FileName))
usnRecordOffsetFileReference = int(unsafe.Offsetof(win32api.USN_RECORD{}.FileReferenceNumber))
usnRecordOffsetParentReference = int(unsafe.Offsetof(win32api.USN_RECORD{}.ParentFileReferenceNumber))
usnRecordOffsetReason = int(unsafe.Offsetof(win32api.USN_RECORD{}.Reason))
usnRecordOffsetFileAttributes = int(unsafe.Offsetof(win32api.USN_RECORD{}.FileAttributes))
usnRecordOffsetFileNameLength = int(unsafe.Offsetof(win32api.USN_RECORD{}.FileNameLength))
usnRecordOffsetFileNameOffset = int(unsafe.Offsetof(win32api.USN_RECORD{}.FileNameOffset))
)
type usnRecordData struct {
FileReferenceNumber win32api.DWORDLONG
ParentFileReferenceNumber win32api.DWORDLONG
Reason win32api.DWORD
FileAttributes win32api.DWORD
FileName string
}
func parseUSNOutput(data []byte, done uint32, fn func(usnRecordData) error) (uint64, error) {
if fn == nil {
return 0, fmt.Errorf("nil USN record callback")
}
if done == 0 {
return 0, nil
}
if done < uint32(usnBufferHeaderSize) {
return 0, fmt.Errorf("USN output too short: %d", done)
}
if int(done) > len(data) {
return 0, fmt.Errorf("USN output length %d exceeds buffer %d", done, len(data))
}
next := binary.LittleEndian.Uint64(data[:usnBufferHeaderSize])
for offset := usnBufferHeaderSize; offset < int(done); {
remaining := int(done) - offset
if remaining < usnRecordMinSize {
return next, fmt.Errorf("USN record header truncated: %d bytes remain", remaining)
}
recordLength := int(binary.LittleEndian.Uint32(data[offset:]))
if recordLength < usnRecordMinSize {
return next, fmt.Errorf("invalid USN record length %d", recordLength)
}
if recordLength > remaining {
return next, fmt.Errorf("USN record length %d exceeds remaining %d", recordLength, remaining)
}
record := data[offset : offset+recordLength]
nameLength := int(binary.LittleEndian.Uint16(record[usnRecordOffsetFileNameLength:]))
nameOffset := int(binary.LittleEndian.Uint16(record[usnRecordOffsetFileNameOffset:]))
if nameLength < 0 || nameLength%2 != 0 {
return next, fmt.Errorf("invalid USN file name length %d", nameLength)
}
if nameOffset < usnRecordMinSize || nameOffset > recordLength {
return next, fmt.Errorf("invalid USN file name offset %d", nameOffset)
}
if nameOffset+nameLength > recordLength {
return next, fmt.Errorf("USN file name exceeds record boundary: offset=%d length=%d record=%d", nameOffset, nameLength, recordLength)
}
name, err := decodeUTF16Bytes(record[nameOffset : nameOffset+nameLength])
if err != nil {
return next, err
}
entry := usnRecordData{
FileReferenceNumber: win32api.DWORDLONG(binary.LittleEndian.Uint64(record[usnRecordOffsetFileReference:])),
ParentFileReferenceNumber: win32api.DWORDLONG(binary.LittleEndian.Uint64(record[usnRecordOffsetParentReference:])),
Reason: win32api.DWORD(binary.LittleEndian.Uint32(record[usnRecordOffsetReason:])),
FileAttributes: win32api.DWORD(binary.LittleEndian.Uint32(record[usnRecordOffsetFileAttributes:])),
FileName: name,
}
if err := fn(entry); err != nil {
return next, err
}
offset += recordLength
}
return next, nil
}
func decodeUTF16Bytes(data []byte) (string, error) {
if len(data)%2 != 0 {
return "", fmt.Errorf("UTF-16 byte length must be even, got %d", len(data))
}
chars := make([]uint16, len(data)/2)
for i := range chars {
chars[i] = binary.LittleEndian.Uint16(data[i*2:])
}
return syscall.UTF16ToString(chars), nil
}
func fileEntryFromUSNRecord(record usnRecordData) FileEntry {
typed := uint8(0)
if record.FileAttributes&win32api.FILE_ATTRIBUTE_DIRECTORY != 0 {
typed = 1
}
return FileEntry{
Name: record.FileName,
Parent: record.ParentFileReferenceNumber,
Type: typed,
}
}
func shouldPreferUSNFileName(current string, candidate string) bool {
if candidate == "" {
return false
}
if current == "" {
return true
}
if strings.EqualFold(current, candidate) {
return false
}
currentShort := strings.Contains(current, "~")
candidateShort := strings.Contains(candidate, "~")
if currentShort != candidateShort {
return currentShort && !candidateShort
}
return len(candidate) > len(current)
}
func mergeUSNFileEntry(current FileEntry, candidate FileEntry) FileEntry {
if current.Name == "" && current.Parent == 0 && current.Type == 0 {
return candidate
}
merged := current
if shouldPreferUSNFileName(merged.Name, candidate.Name) {
merged.Name = candidate.Name
}
if candidate.Name != "" && !strings.EqualFold(merged.Name, candidate.Name) && !shouldPreferUSNFileName(candidate.Name, merged.Name) {
merged.Name = candidate.Name
}
if merged.Name == "" {
merged.Name = candidate.Name
}
if candidate.Parent != 0 {
merged.Parent = candidate.Parent
}
if candidate.Type == 1 {
merged.Type = 1
}
return merged
}
func needPathCanonicalNameOverlay(fileMap map[win32api.DWORDLONG]FileEntry) bool {
for _, entry := range fileMap {
if strings.Contains(entry.Name, "~") {
return true
}
}
return false
}
func windowsBaseName(path string) string {
trimmed := strings.TrimRight(path, `\/`)
if trimmed == "" {
return ""
}
last := strings.LastIndexAny(trimmed, `\/`)
if last < 0 {
return trimmed
}
return trimmed[last+1:]
}
func applyPathCanonicalNames(driver string, fileMap map[win32api.DWORDLONG]FileEntry) {
if len(fileMap) == 0 || !needPathCanonicalNameOverlay(fileMap) {
return
}
for id, entry := range fileMap {
if !strings.Contains(entry.Name, "~") {
continue
}
path := buildUSNPath(driver, fileMap, id)
normalized := normalizePathForUSN(path)
base := windowsBaseName(normalized)
if base == "" {
continue
}
entry.Name = base
fileMap[id] = entry
}
}
func buildUSNFileMap(driver string) (map[win32api.DWORDLONG]FileEntry, error) {
fileMap := make(map[win32api.DWORDLONG]FileEntry)
pDriver := "\\\\.\\" + driver[:len(driver)-1]
pDriver, err := volumeDevicePath(driver)
if err != nil {
return fileMap, err
}
fd, err := CreateFile(pDriver, syscall.O_RDONLY, win32api.FILE_ATTRIBUTE_NORMAL)
if err != nil {
return fileMap, err
}
defer syscall.Close(fd)
ujd, _, err := queryUsnJournal(fd)
if err != nil {
return fileMap, err
@@ -197,77 +433,51 @@ func ListUsnFile(driver string) (map[win32api.DWORDLONG]FileEntry, error) {
return fileMap, err
}
if done == 0 {
applyPathCanonicalNames(driver, fileMap)
return fileMap, nil
}
var usn win32api.USN = *(*win32api.USN)(unsafe.Pointer(&data[0]))
// fmt.Println("usn", usn)
var ur *win32api.USN_RECORD
for i := unsafe.Sizeof(usn); i < uintptr(done); i += uintptr(ur.RecordLength) {
ur = (*win32api.USN_RECORD)(unsafe.Pointer(&data[i]))
nameLength := uintptr(ur.FileNameLength) / unsafe.Sizeof(ur.FileName[0])
fnp := unsafe.Pointer(&data[i+uintptr(ur.FileNameOffset)])
fnUtf := (*[10000]uint16)(fnp)[:nameLength]
fn := syscall.UTF16ToString(fnUtf)
(*reflect.SliceHeader)(unsafe.Pointer(&fn)).Cap = int(nameLength)
typed := uint8(0)
if ur.FileAttributes&win32api.FILE_ATTRIBUTE_DIRECTORY != 0 {
typed = 1
}
// fmt.Println("len", ur.FileNameLength, ur.FileNameOffset, "fn", fn)
fileMap[ur.FileReferenceNumber] = FileEntry{Name: fn, Parent: ur.ParentFileReferenceNumber, Type: typed}
nextRef, err := parseUSNOutput(data, done, func(record usnRecordData) error {
fileMap[record.FileReferenceNumber] = mergeUSNFileEntry(fileMap[record.FileReferenceNumber], fileEntryFromUSNRecord(record))
return nil
})
if err != nil {
return fileMap, err
}
med.StartFileReferenceNumber = win32api.DWORDLONG(usn)
med.StartFileReferenceNumber = win32api.DWORDLONG(nextRef)
}
}
func filterUSNFileMap(fileMap map[win32api.DWORDLONG]FileEntry, searchFn func(string, bool) bool) map[win32api.DWORDLONG]FileEntry {
if searchFn == nil {
return fileMap
}
filtered := make(map[win32api.DWORDLONG]FileEntry)
for id, entry := range fileMap {
if entry.Type == 1 || searchFn(entry.Name, entry.Type == 1) {
filtered[id] = entry
}
}
return filtered
}
func ListUsnFile(driver string) (map[win32api.DWORDLONG]FileEntry, error) {
return buildUSNFileMap(driver)
}
func ListUsnFileFn(driver string, searchFn func(string, bool) bool) (map[win32api.DWORDLONG]FileEntry, error) {
fileMap := make(map[win32api.DWORDLONG]FileEntry)
pDriver := "\\\\.\\" + driver[:len(driver)-1]
fd, err := CreateFile(pDriver, syscall.O_RDONLY, win32api.FILE_ATTRIBUTE_NORMAL)
fileMap, err := buildUSNFileMap(driver)
if err != nil {
return fileMap, err
}
ujd, _, err := queryUsnJournal(fd)
if err != nil {
return fileMap, err
}
med := win32api.MFT_ENUM_DATA{0, 0, ujd.NextUsn}
for {
data, done, err := enumUsnData(fd, &med)
if err != nil && done != 0 {
return fileMap, err
}
if done == 0 {
return fileMap, nil
}
var usn win32api.USN = *(*win32api.USN)(unsafe.Pointer(&data[0]))
// fmt.Println("usn", usn)
var ur *win32api.USN_RECORD
for i := unsafe.Sizeof(usn); i < uintptr(done); i += uintptr(ur.RecordLength) {
ur = (*win32api.USN_RECORD)(unsafe.Pointer(&data[i]))
nameLength := uintptr(ur.FileNameLength) / unsafe.Sizeof(ur.FileName[0])
fnp := unsafe.Pointer(&data[i+uintptr(ur.FileNameOffset)])
fnUtf := (*[10000]uint16)(fnp)[:nameLength]
fn := syscall.UTF16ToString(fnUtf)
(*reflect.SliceHeader)(unsafe.Pointer(&fn)).Cap = int(nameLength)
typed := uint8(0)
if ur.FileAttributes&win32api.FILE_ATTRIBUTE_DIRECTORY != 0 {
typed = 1
}
if typed == 1 || searchFn(fn, typed == 1) {
// fmt.Println("len", ur.FileNameLength, ur.FileNameOffset, "fn", fn)
fileMap[ur.FileReferenceNumber] = FileEntry{Name: fn, Parent: ur.ParentFileReferenceNumber, Type: typed}
}
}
med.StartFileReferenceNumber = win32api.DWORDLONG(usn)
}
return filterUSNFileMap(fileMap, searchFn), nil
}
func GetFullUsnPath(diskName string, fileMap map[win32api.DWORDLONG]FileEntry, id win32api.DWORDLONG) (name string) {
func buildUSNPath(diskName string, fileMap map[win32api.DWORDLONG]FileEntry, id win32api.DWORDLONG) (name string) {
normalized, err := normalizeDiskName(diskName)
if err != nil {
return ""
}
for id != 0 {
fe := fileMap[id]
if id == fe.Parent {
@@ -281,32 +491,139 @@ func GetFullUsnPath(diskName string, fileMap map[win32api.DWORDLONG]FileEntry, i
}
id = fe.Parent
}
name = diskName[:len(diskName)-1] + name
name = strings.TrimSuffix(normalized, "\\") + name
return
}
func GetFullUsnPathEntry(diskName string, fileMap map[win32api.DWORDLONG]FileEntry, en FileMonitor) (name string) {
fileMap[en.Self] = FileEntry{
func normalizeExistingLongPath(path string) string {
if path == "" {
return path
}
if normalized, ok := getLongPathName(path); ok {
return trimLongPathPrefix(normalized)
}
longPath := fixLongPath(path)
if longPath == path {
return path
}
if normalized, ok := getLongPathName(longPath); ok {
return trimLongPathPrefix(normalized)
}
return path
}
func getLongPathName(path string) (string, bool) {
pathp, err := syscall.UTF16PtrFromString(path)
if err != nil {
return "", false
}
size := len(path) + 1
if size < syscall.MAX_PATH {
size = syscall.MAX_PATH
}
for {
buf := make([]uint16, size)
n, err := syscall.GetLongPathName(pathp, &buf[0], uint32(len(buf)))
if err != nil || n == 0 {
return "", false
}
if int(n) < len(buf) {
return syscall.UTF16ToString(buf[:n]), true
}
size = int(n) + 1
}
}
func trimLongPathPrefix(path string) string {
switch {
case strings.HasPrefix(path, `\\?\UNC\`):
return `\\` + path[len(`\\?\UNC\`):]
case strings.HasPrefix(path, `\\?\`):
return path[len(`\\?\`):]
default:
return path
}
}
func GetFullUsnPath(diskName string, fileMap map[win32api.DWORDLONG]FileEntry, id win32api.DWORDLONG) string {
return normalizeExistingLongPath(buildUSNPath(diskName, fileMap, id))
}
func GetFullUsnPathEntry(diskName string, fileMap map[win32api.DWORDLONG]FileEntry, en FileMonitor) string {
fileMap[en.Self] = mergeUSNFileEntry(fileMap[en.Self], FileEntry{
Name: en.Name,
Parent: en.Parent,
Type: en.Type,
})
return normalizeExistingLongPath(buildUSNPath(diskName, fileMap, en.Self))
}
func fileStatFromHandle(fd syscall.Handle, name string, path string) (FileStat, error) {
var info syscall.ByHandleFileInformation
if err := syscall.GetFileInformationByHandle(fd, &info); err != nil {
return FileStat{}, err
}
id := en.Self
for id != 0 {
fe := fileMap[id]
if id == fe.Parent {
name = "\\" + name
break
}
if name == "" {
name = fe.Name
} else {
name = fe.Name + "\\" + name
}
id = fe.Parent
stat := newFileStatFromInformation(&info, name, path)
fileType, err := syscall.GetFileType(fd)
if err == nil {
stat.filetype = fileType
}
name = diskName[:len(diskName)-1] + name
return
return stat, nil
}
func fileStatFromPath(name string, path string) (FileStat, error) {
fileInfo, err := os.Stat(path)
if err != nil {
return FileStat{}, err
}
data, ok := fileInfo.Sys().(*syscall.Win32FileAttributeData)
if !ok {
return FileStat{}, fmt.Errorf("unexpected file info payload %T", fileInfo.Sys())
}
return FileStat{
name: name,
path: path,
FileAttributes: data.FileAttributes,
CreationTime: data.CreationTime,
LastAccessTime: data.LastAccessTime,
LastWriteTime: data.LastWriteTime,
FileSizeHigh: data.FileSizeHigh,
FileSizeLow: data.FileSizeLow,
}, nil
}
func fileOpenAttributes(entryType uint8) uint32 {
if entryType == 1 {
return win32api.FILE_FLAG_BACKUP_SEMANTICS
}
return win32api.FILE_ATTRIBUTE_NORMAL
}
func fileStatFromIDWithfd(volumeHandle syscall.Handle, id win32api.DWORDLONG, name string, path string, entryType uint8) (FileStat, error) {
fileHandle, err := OpenFileByIdWithfd(volumeHandle, id, syscall.O_RDONLY, fileOpenAttributes(entryType))
if err != nil {
return FileStat{}, err
}
defer syscall.Close(fileHandle)
return fileStatFromHandle(fileHandle, name, path)
}
func fileStatForEntryWithfd(volumeHandle syscall.Handle, diskName string, data map[win32api.DWORDLONG]FileEntry, id win32api.DWORDLONG, entry FileEntry) (FileStat, error) {
path := GetFullUsnPath(diskName, data, id)
stat, err := fileStatFromIDWithfd(volumeHandle, id, entry.Name, path, entry.Type)
if err == nil {
return stat, nil
}
fallback, fallbackErr := fileStatFromPath(entry.Name, path)
if fallbackErr == nil {
return fallback, nil
}
return FileStat{}, fmt.Errorf("stat by id: %v; stat by path: %w", err, fallbackErr)
}
func fileStatForEntryByPath(diskName string, data map[win32api.DWORDLONG]FileEntry, id win32api.DWORDLONG, entry FileEntry) (FileStat, error) {
path := GetFullUsnPath(diskName, data, id)
return fileStatFromPath(entry.Name, path)
}
const (
@@ -352,12 +669,7 @@ func listNTFSUsnDriverFiles(diskName string, fn func(string, bool) bool, data ma
result[i] = name
i++
}
(*reflect.SliceHeader)(unsafe.Pointer(&result)).Cap = i
(*reflect.SliceHeader)(unsafe.Pointer(&result)).Len = i
data = nil
data = make(map[win32api.DWORDLONG]FileEntry, 0)
runtime.GC()
return result, nil
return result[:i], nil
}
func ListNTFSUsnDriverInfoFn(diskName string, searchFn func(string, bool) bool) ([]FileStat, error) {
@@ -384,73 +696,67 @@ func ListNTFSUsnDriverInfo(diskName string, folder uint8) ([]FileStat, error) {
}, data)
}
func listNTFSUsnDriverInfo(diskName string, fn func(string, bool) bool, data map[win32api.DWORDLONG]FileEntry) ([]FileStat, error) {
//fmt.Println("finished 1")
pDriver := "\\\\.\\" + diskName[:len(diskName)-1]
fd, err := CreateFile(pDriver, syscall.O_RDONLY, win32api.FILE_ATTRIBUTE_NORMAL)
if err != nil {
return nil, err
type fileStatFetcher func(id win32api.DWORDLONG, entry FileEntry) (FileStat, error)
func collectUSNFileStats(data map[win32api.DWORDLONG]FileEntry, fn func(string, bool) bool, fetch fileStatFetcher) []FileStat {
if fetch == nil {
return []FileStat{}
}
defer syscall.Close(fd)
result := make([]FileStat, len(data))
i := int(0)
if fn == nil {
fn = func(string, bool) bool { return true }
}
resultCh := make(chan FileStat, len(data))
wg := stario.NewWaitGroup(100)
for k, v := range data {
if !fn(v.Name, v.Type == 1) {
for id, entry := range data {
if !fn(entry.Name, entry.Type == 1) {
continue
}
wg.Add(1)
go func(k win32api.DWORDLONG, v FileEntry, i int) {
go func(id win32api.DWORDLONG, entry FileEntry) {
defer wg.Done()
//now := time.Now().UnixNano()
/*
fd2, err := OpenFileByIdWithfd(fd, k, syscall.O_RDONLY, win32api.FILE_ATTRIBUTE_NORMAL)
if err != nil {
return
}
//fmt.Println("cost", float64((time.Now().UnixNano()-now)/1000000))
var info syscall.ByHandleFileInformation
err = syscall.GetFileInformationByHandle(fd2, &info)
syscall.Close(fd2)
//fmt.Println("cost", float64((time.Now().UnixNano()-now)/1000000))
if err != nil {
return
}
*/
path := GetFullUsnPath(diskName, data, k)
fileInfo, err := os.Stat(path)
stat, err := fetch(id, entry)
if err != nil {
return
}
fs := fileInfo.Sys().(*syscall.Win32FileAttributeData)
stat := FileStat{
FileAttributes: fs.FileAttributes,
CreationTime: fs.CreationTime,
LastAccessTime: fs.LastAccessTime,
LastWriteTime: fs.LastWriteTime,
FileSizeHigh: fs.FileSizeHigh,
FileSizeLow: fs.FileSizeLow,
}
stat.name = v.Name
stat.path = path
return
result[i] = stat
//result[i] = newFileStatFromInformation(&info, v.Name, path)
}(k, v, i)
i++
resultCh <- stat
}(id, entry)
}
wg.Wait()
//fmt.Println("finished 2")
(*reflect.SliceHeader)(unsafe.Pointer(&result)).Cap = i
(*reflect.SliceHeader)(unsafe.Pointer(&result)).Len = i
data = nil
//data = make(map[win32api.DWORDLONG]FileEntry, 0)
runtime.GC()
return result, nil
close(resultCh)
result := make([]FileStat, 0, len(data))
for stat := range resultCh {
result = append(result, stat)
}
return result
}
func getUsnJournalReasonString(reason win32api.DWORD) (s string) {
func listNTFSUsnDriverInfo(diskName string, fn func(string, bool) bool, data map[win32api.DWORDLONG]FileEntry) ([]FileStat, error) {
pDriver, err := volumeDevicePath(diskName)
if err != nil {
return nil, err
}
fd, err := CreateFile(pDriver, syscall.O_RDONLY, win32api.FILE_ATTRIBUTE_NORMAL)
useByID := err == nil
if useByID {
defer syscall.Close(fd)
}
var fetch fileStatFetcher
if useByID {
fetch = func(id win32api.DWORDLONG, entry FileEntry) (FileStat, error) {
return fileStatForEntryWithfd(fd, diskName, data, id, entry)
}
} else {
fetch = func(id win32api.DWORDLONG, entry FileEntry) (FileStat, error) {
return fileStatForEntryByPath(diskName, data, id, entry)
}
}
return collectUSNFileStats(data, fn, fetch), nil
}
func USNReasonString(reason win32api.DWORD) (s string) {
var reasons = []string{
"DataOverwrite", // 0x00000001
"DataExtend", // 0x00000002
@@ -485,75 +791,84 @@ func getUsnJournalReasonString(reason win32api.DWORD) (s string) {
"0x40000000", // 0x40000000
"*Close*", // 0x80000000
}
for i := 0; reason != 0; {
for i := 0; reason != 0; i++ {
if i >= len(reasons) {
if s == "" {
return fmt.Sprintf("0x%08X", uint32(reason)<<uint(i))
}
return s + fmt.Sprintf(", 0x%08X", uint32(reason)<<uint(i))
}
if reason&1 == 1 {
s = s + ", " + reasons[i]
}
reason >>= 1
i++
}
return
}
func getUsnJournalReasonString(reason win32api.DWORD) string {
return USNReasonString(reason)
}
func MonitorUsnChange(driver string, rec chan FileMonitor) error {
pDriver := "\\\\.\\" + driver[:len(driver)-1]
pDriver, err := volumeDevicePath(driver)
if err != nil {
return err
}
fd, err := CreateFile(pDriver, syscall.O_RDONLY, win32api.FILE_ATTRIBUTE_NORMAL)
if err != nil {
return err
}
defer syscall.Close(fd)
ujd, _, err := queryUsnJournal(fd)
if err != nil {
return err
}
rujd := win32api.READ_USN_JOURNAL_DATA{ujd.NextUsn, 0xFFFFFFFF, 0, 0, 1, ujd.UsnJournalID}
cache := make(map[win32api.DWORDLONG]FileEntry)
for {
var usn win32api.USN
data, done, err := readUsnJournal(fd, &rujd)
if err != nil || done <= uint32(unsafe.Sizeof(usn)) {
if err != nil || done <= uint32(usnBufferHeaderSize) {
return err
}
usn = *(*win32api.USN)(unsafe.Pointer(&data[0]))
var ur *win32api.USN_RECORD
for i := unsafe.Sizeof(usn); i < uintptr(done); i += uintptr(ur.RecordLength) {
ur = (*win32api.USN_RECORD)(unsafe.Pointer(&data[i]))
nameLength := uintptr(ur.FileNameLength) / unsafe.Sizeof(ur.FileName[0])
fnp := unsafe.Pointer(&data[i+uintptr(ur.FileNameOffset)])
fn := syscall.UTF16ToString((*[10000]uint16)(fnp)[:nameLength])
(*reflect.SliceHeader)(unsafe.Pointer(&fn)).Cap = int(nameLength)
// fmt.Println("len", ur.FileNameLength, ur.FileNameOffset, "fn", getFullPath(folders, ur.ParentFileReferenceNumber), syscall.UTF16ToString(fn), getUsnJournalReasonString(ur.Reason))
typed := uint8(0)
if ur.FileAttributes&win32api.FILE_ATTRIBUTE_DIRECTORY != 0 {
typed = 1
}
// fmt.Println("len", ur.FileNameLength, ur.FileNameOffset, "fn", fn)
rec <- FileMonitor{Name: fn, Parent: ur.ParentFileReferenceNumber, Type: typed, Self: ur.FileReferenceNumber, Reason: getUsnJournalReasonString(ur.Reason)}
nextUsn, err := parseUSNOutput(data, done, func(record usnRecordData) error {
entry := mergeUSNFileEntry(cache[record.FileReferenceNumber], fileEntryFromUSNRecord(record))
cache[record.FileReferenceNumber] = entry
rec <- FileMonitor{Name: entry.Name, Parent: entry.Parent, Type: entry.Type, Self: record.FileReferenceNumber, Reason: getUsnJournalReasonString(record.Reason)}
return nil
})
if err != nil {
return err
}
rujd.StartUsn = usn
if usn == 0 {
rujd.StartUsn = win32api.USN(nextUsn)
if nextUsn == 0 {
return nil
}
}
}
func GetUsnFileInfo(diskName string, fileMap map[win32api.DWORDLONG]FileEntry, id win32api.DWORDLONG) (FileStat, error) {
name := fileMap[id].Name
path := GetFullUsnPath(diskName, fileMap, id)
fd, err := OpenFileById(diskName, id, syscall.O_RDONLY, win32api.FILE_ATTRIBUTE_NORMAL)
pDriver, err := volumeDevicePath(diskName)
if err != nil {
return FileStat{}, err
}
var info syscall.ByHandleFileInformation
err = syscall.GetFileInformationByHandle(fd, &info)
return newFileStatFromInformation(&info, name, path), err
volumeHandle, err := CreateFile(pDriver, syscall.O_RDONLY, win32api.FILE_ATTRIBUTE_NORMAL)
if err != nil {
return fileStatForEntryByPath(diskName, fileMap, id, fileMap[id])
}
defer syscall.Close(volumeHandle)
return fileStatForEntryWithfd(volumeHandle, diskName, fileMap, id, fileMap[id])
}
// Need a custom Open to work with backup_semantics
func OpenFileById(diskName string, id win32api.DWORDLONG, mode int, attrs uint32) (syscall.Handle, error) {
pDriver := "\\\\.\\" + diskName[:len(diskName)-1]
pDriver, err := volumeDevicePath(diskName)
if err != nil {
return syscall.InvalidHandle, err
}
fd, err := CreateFile(pDriver, syscall.O_RDONLY, win32api.FILE_ATTRIBUTE_NORMAL)
if err != nil {
return syscall.InvalidHandle, err
@@ -585,11 +900,10 @@ func OpenFileByIdWithfd(fd syscall.Handle, id win32api.DWORDLONG, mode int, attr
sa = makeInheritSa()
}
fid := win32api.FILE_ID_DESCRIPTOR{
DwSize: 16,
Type: 0,
DwSize: win32api.DWORD(unsafe.Sizeof(win32api.FILE_ID_DESCRIPTOR{})),
Type: win32api.FileIdType,
FileId: id,
}
fid.DwSize = win32api.DWORD(unsafe.Sizeof(fid))
h, e := win32api.OpenFileById(win32api.HANDLE(fd), &fid, win32api.DWORD(access),
win32api.DWORD(sharemode), sa, win32api.DWORD(attrs))
return syscall.Handle(h), e