Asynchronous NtQueryDirectoryFile?

Has anyone managed to figure out how asynchronous calls to NtQueryDirectoryFile work?

By an asynchronous call, I mean calling NtQueryDirectoryFile on directories not开发者_如何学JAVA opened with FILE_SYNCHRONOUS_IO_ALERT or with FILE_SYNCHRONOUS_IO_NONALERT.

For me, it seems to return STATUS_PENDING just like a normal NtReadFile request does on a normal file, but when I tried using NtWaitForSingleObject on the directory, it didn't end properly, and I still don't get all the data... why does this happen?

As far as I know, none of the Windows filesystems support asynchronous query directory calls. The Win32 APIs never call NtQueryDirectoryFile asnchronously, so support for it is hit-or-miss.

NTFS theoretically supports asynchronous NtQueryDirectoryFile but (as I mentioned) it is not extensively tested so it may not work.

You response indicated that you called WaitForSingleObject on the directory - that's not how the async pattern works in NT - you need to call WaitForSingleObject on the event handle provided as a parameter to NtQueryDirectoryFile.

This update is a result of asking the NTFS developer for more information, he tested this scenario on his machine and it worked for him (on Windows 7).

NtQueryDirectoryFile works well in asynchronous!

pass callback in ApcRoutine, and callback data in ApcContext

asynchronous procedure calls only call when the thread is in alertable state(for example: calling SleepEx(INFINITE, TRUE), WSAaccept)

this program shows how asynchronous NtQueryDirectoryFile work.

#define WIN32_LEAN_AND_MEAN
#include <Windows.h>
#include <stdio.h>
#include <winternl.h>
#include <winnt.h>

#define LIST_DIR_SIZE 2000
#define STATUS_NO_MORE_FILES ((NTSTATUS)80000006)

typedef struct _FILE_NAMES_INFORMATION {
    ULONG NextEntryOffset;
    ULONG FileIndex;
    ULONG FileNameLength;
    WCHAR FileName[1];
} FILE_NAMES_INFORMATION, * PFILE_NAMES_INFORMATION;

typedef struct {
    HANDLE hFile;
    OVERLAPPED ol;
    DECLSPEC_ALIGN(4) FILE_NAMES_INFORMATION buf[LIST_DIR_SIZE];
    IO_STATUS_BLOCK iob;
    bool finished;
} LIST_DIR_DATA, * PLIST_DIR_DATA; // my private data


__kernel_entry NTSYSCALLAPI
NTSTATUS
NTAPI
NtQueryDirectoryFile(
    _In_ HANDLE FileHandle,
    _In_opt_ HANDLE Event,
    _In_opt_ PIO_APC_ROUTINE ApcRoutine,
    _In_opt_ PVOID ApcContext,
    _Out_ PIO_STATUS_BLOCK IoStatusBlock,
    _Out_writes_bytes_(Length) PVOID FileInformation,
    _In_ ULONG Length,
    _In_ FILE_INFORMATION_CLASS FileInformationClass,
    _In_ BOOLEAN ReturnSingleEntry,
    _In_opt_ PUNICODE_STRING FileName,
    _In_ BOOLEAN RestartScan
);

#define NTDLL_extern(s) typedef decltype(&s) s##T;s##T s##F;
#define NTDLL_import(s) s##F = (s##T)GetProcAddress(ntdll, #s);

NTDLL_extern(NtOpenFile);
NTDLL_extern(NtQueryDirectoryFile);
NTDLL_extern(NtClose);
NTDLL_extern(RtlInitUnicodeString);

HMODULE ntdll;

VOID NTAPI callback(
    IN PVOID ApcContext,
    IN PIO_STATUS_BLOCK IoStatusBlock,
    IN ULONG Reserved) {

    UNREFERENCED_PARAMETER(Reserved);
    PFILE_NAMES_INFORMATION file_info = ((PLIST_DIR_DATA)ApcContext)->buf;
    do {
        fputws(file_info->FileName, stdout);
        putwchar(L'\t');
        file_info = (PFILE_NAMES_INFORMATION)((char*)file_info + file_info->NextEntryOffset);
    } while (file_info->NextEntryOffset);
    fputws(file_info->FileName, stdout);
    putwchar(L'\t');
    PLIST_DIR_DATA c = (PLIST_DIR_DATA)ApcContext;
    if (IoStatusBlock->Information != 0) {
        NTSTATUS status = NtQueryDirectoryFileF(
            c->hFile,
            NULL,
            callback,
            ApcContext,
            &c->iob,
            c->buf,
            sizeof(c->buf),
            FILE_INFORMATION_CLASS(12),
            FALSE, NULL, FALSE);
        switch (status) {
        case STATUS_PENDING:
            break;
        default:
            fputs("warning: status != STATUS_PENDING", stderr);
        }
    }
    else {
        c->finished = true;
    }
}
BOOL init() {
    ntdll = LoadLibraryW(L"NtDLL.dll");
    if (ntdll == NULL) {
        fputs("LoadLibraryW", stderr);
        return FALSE;
    }
    NTDLL_import(NtQueryDirectoryFile);
    NTDLL_import(NtOpenFile);
    NTDLL_import(NtClose);
    NTDLL_import(RtlInitUnicodeString);
    if (NtCloseF != NULL && NtOpenFileF != NULL && NtCloseF != NULL) {
        return TRUE;
    }
    else {
        fputs("GetProcAddress", stderr);
        return FALSE;
    }
}

int main() {
    if (init() == FALSE) {
        fputs("error: init() failed!", stderr);
        return -1;
    }
    NTSTATUS status;
    PLIST_DIR_DATA data = new LIST_DIR_DATA{};
    {
        OBJECT_ATTRIBUTES ObjectAttributes;
        UNICODE_STRING s;
        RtlInitUnicodeStringF(&s, L"\\??\\c:\\Windows\\System32");
        InitializeObjectAttributes(
            &ObjectAttributes,
            &s,
            OBJ_CASE_INSENSITIVE,
            NULL,
            NULL);
        status = NtOpenFileF(
            &data->hFile,
            FILE_READ_DATA | FILE_LIST_DIRECTORY, // | FILE_TRAVERSE | SYNCHRONIZE
            &ObjectAttributes,
            &data->iob,
            FILE_SHARE_READ,
            FILE_DIRECTORY_FILE); // | FILE_SYNCHRONOUS_IO_NONALERT
    }
    if (status < 0 || data->hFile == NULL) {
        fputs("error: NtOpenFile failed", stderr);
        return -2;
    }
    status = NtQueryDirectoryFileF(
        data->hFile,
        NULL,
        callback,
        data,
        &data->iob,
        data->buf,
        sizeof(data->buf),
        FILE_INFORMATION_CLASS(12),
        FALSE, NULL, FALSE);
    switch (status) {
        case STATUS_PENDING: 
            break;
        default:
            fputs("warning: status != STATUS_PENDING", stderr);
    }
    for (;data->finished==false;) SleepEx(INFINITE, TRUE); // put main thread into alertable wait

    NtCloseF(data->hFile);
    FreeLibrary(ntdll);
    return 0;
}

if you want UTF-8 output, try this (note: recommand use support UTF-8 terminal)

#define WIN32_LEAN_AND_MEAN
#include <Windows.h>
#include <winternl.h>
#include <winnt.h>
#include <crtdbg.h>
#include <cstdio>

#define LIST_DIR_SIZE 200
#define STATUS_NO_MORE_FILES ((NTSTATUS)80000006)

typedef struct _FILE_NAMES_INFORMATION {
    ULONG NextEntryOffset;
    ULONG FileIndex;
    ULONG FileNameLength;
    WCHAR FileName[1];
} FILE_NAMES_INFORMATION, * PFILE_NAMES_INFORMATION;

typedef struct {
    HANDLE hFile;
    OVERLAPPED ol;
    DECLSPEC_ALIGN(4) FILE_NAMES_INFORMATION buf[LIST_DIR_SIZE];
    IO_STATUS_BLOCK iob;
    bool finished;
} LIST_DIR_DATA, * PLIST_DIR_DATA; // my private data


__kernel_entry NTSYSCALLAPI
NTSTATUS
NTAPI
NtQueryDirectoryFile(
    _In_ HANDLE FileHandle,
    _In_opt_ HANDLE Event,
    _In_opt_ PIO_APC_ROUTINE ApcRoutine,
    _In_opt_ PVOID ApcContext,
    _Out_ PIO_STATUS_BLOCK IoStatusBlock,
    _Out_writes_bytes_(Length) PVOID FileInformation,
    _In_ ULONG Length,
    _In_ FILE_INFORMATION_CLASS FileInformationClass,
    _In_ BOOLEAN ReturnSingleEntry,
    _In_opt_ PUNICODE_STRING FileName,
    _In_ BOOLEAN RestartScan
);

#define NTDLL_extern(s) typedef decltype(&s) s##T;s##T s##F;
#define NTDLL_init(s) s##F = (s##T)GetProcAddress(ntdll, #s);

NTDLL_extern(NtOpenFile);
NTDLL_extern(NtQueryDirectoryFile);
NTDLL_extern(NtClose);
NTDLL_extern(RtlInitUnicodeString);

HMODULE ntdll;
HANDLE heap;

VOID NTAPI callback(
    IN PVOID ApcContext,
    IN PIO_STATUS_BLOCK IoStatusBlock,
    IN ULONG Reserved) {
    UNREFERENCED_PARAMETER(Reserved);
    PLIST_DIR_DATA c = (PLIST_DIR_DATA)ApcContext;
    if (IoStatusBlock->Information){
        PFILE_NAMES_INFORMATION file_info = c->buf;
        ULONG_PTR length = 0;
        ULONG last;
        do {
            last = file_info->NextEntryOffset;
            file_info->FileNameLength /= 2; // wide char length always base of 2 in bytes
            length += (
                file_info->FileIndex=WideCharToMultiByte(
                    CP_UTF8, WC_ERR_INVALID_CHARS, 
                    file_info->FileName, file_info->FileNameLength, 
                    NULL, 0, 
                    NULL, NULL)
                )+1;
            if (file_info->FileIndex == 0) { // FileIndex is how many byte is the UTF-8 string
                _RPTF0(_CRT_WARN, "WideCharToMultiByte failed!");
            }
            file_info = (PFILE_NAMES_INFORMATION)((char*)file_info + file_info->NextEntryOffset);
        } while (last);
        LPSTR pData = (LPSTR)HeapAlloc(heap, HEAP_NO_SERIALIZE, length), ptr=pData;
        if (ptr == NULL) {
            _RPTF0(_CRT_ERROR, "HeapAlloc failed!");
            return;
        }
        file_info = c->buf;
        do {
            last = file_info->NextEntryOffset;
            if (WideCharToMultiByte(
                CP_UTF8, WC_ERR_INVALID_CHARS,
                file_info->FileName, file_info->FileNameLength, 
                pData, file_info->FileIndex, 
                NULL, NULL)==0) {
                _RPTF0(_CRT_WARN, "WideCharToMultiByte failed!");
            }
            pData += file_info->FileIndex;
            *pData++ = '\n';
            file_info = (PFILE_NAMES_INFORMATION)((char*)file_info + file_info->NextEntryOffset);
        } while (last);

        // use data here
        fwrite(ptr, length, 1, stdout);
        // use data here

        HeapFree(heap, HEAP_NO_SERIALIZE, ptr);
        NTSTATUS status = NtQueryDirectoryFileF(
            c->hFile,
            NULL,
            callback,
            ApcContext,
            &c->iob,
            c->buf,
            sizeof(c->buf),
            FILE_INFORMATION_CLASS(12),
            FALSE, NULL, FALSE);

        switch (status) {
        case STATUS_PENDING:
            break;
        default:
            _RPTF0(_CRT_WARN, "status != STATUS_PENDING");
        }
    }else{
        c->finished = true;
    }
}
BOOL init() {
    ntdll = LoadLibraryW(L"NtDLL.dll");
    if (ntdll == NULL) {
        _RPTF0(_CRT_ERROR, "fail to load NtDLL.dll");
        return FALSE;
    }
    NTDLL_init(NtQueryDirectoryFile);
    NTDLL_init(NtOpenFile);
    NTDLL_init(NtClose);
    NTDLL_init(RtlInitUnicodeString);
    if (NtCloseF != NULL && 
        NtOpenFileF != NULL && 
        NtCloseF != NULL && 
        (heap = HeapCreate(HEAP_NO_SERIALIZE, 4096,0))!=NULL
        ){
        return TRUE;
    }
    else {
        _RPTF0(_CRT_ERROR, "failed to load function and create heap");
        return FALSE;
    }
}

int main() {
    if (init() == FALSE) {
        _RPTF0(_CRT_ERROR, "init failed");
        return -1;
    }
    SetConsoleCP(CP_UTF8);
    NTSTATUS status;
    PLIST_DIR_DATA data = new LIST_DIR_DATA{};
    {
        OBJECT_ATTRIBUTES ObjectAttributes;
        UNICODE_STRING s;
        RtlInitUnicodeStringF(&s, L"\\??\\c:\\Users");
        InitializeObjectAttributes(
            &ObjectAttributes,
            &s,
            OBJ_CASE_INSENSITIVE,
            NULL,
            NULL);
        status = NtOpenFileF(
            &data->hFile,
            FILE_READ_DATA | FILE_LIST_DIRECTORY, // | FILE_TRAVERSE | SYNCHRONIZE
            &ObjectAttributes,
            &data->iob,
            FILE_SHARE_READ,
            FILE_DIRECTORY_FILE); // | FILE_SYNCHRONOUS_IO_NONALERT
    }
    if (status < 0 || data->hFile == NULL) {
        _RPTF0(_CRT_ERROR, "NtOpenFile failed!");
        return -2;
    }
    status = NtQueryDirectoryFileF(
        data->hFile,
        NULL,
        callback,
        data,
        &data->iob,
        data->buf,
        sizeof(data->buf),
        FILE_INFORMATION_CLASS(12),
        FALSE, NULL, FALSE);
    switch (status) {
        case STATUS_PENDING: 
            break;
        default:
            _RPTF0(_CRT_WARN, "status != STATUS_PENDING");
    }
    for (;data->finished==false;) SleepEx(INFINITE, TRUE); // put main thread into alertable wait

    if (NtCloseF(data->hFile)<0) {
        _RPTF0(_CRT_ERROR, "NtClose failed!");
    }
    if (FreeLibrary(ntdll) == FALSE) {
        _RPTF0(_CRT_WARN, "failed to Free libary");
    }
    if (HeapDestroy(heap) == FALSE) {
        _RPTF0(_CRT_WARN, "fail to destroy heap");
    }
}