multi-thread opening file hangs when fanotify is on

I use this fanotify sample to monitor open/access perms on the whole file system(/): http://git.infradead.org/users/eparis/fanotify-example.git.

Then I have a test program with multiple threads, each thread iterate the sample foder and open/close the files in it, sometimes my program hangs at open().

OS: Ubuntu 2.6.38-11 x86_64.

Is it a bug of fanotify that it does not support multiple-thread opening?

The code of my test program:

#include <stdlib.h>
#include <stdio.h>
#include <pthread.h>
#include <ctype.h>
#include <unistd.h>
#include <dirent.h>
#include <string.h>
#include <sys/stat.h>
#include <errno.h>
#include <fcntl.h>
#include <dirent.h>

//open f开发者_如何学JAVAile function    
void open_file( char* file )
{
    int fd = -1;
    fd = open( file, O_WRONLY, 0x666 );
    if( fd >= 0 )
    {
        printf("open:%s\n", file );
        close( fd );
    }
}

//iterate directory function

void printdir(char *dir, int depth)
{
    DIR *dp;
    struct stat statbuf;
    char pathbuf[2048] = {0};
    struct dirent entry;
    struct dirent *entryPtr = NULL;

    //printf("opendir %s\n", dir );
    usleep( 300 );
    if((dp = opendir(dir)) == NULL) {
        if( errno != ENOTDIR )
        {
            fprintf(stderr,"cannot open directory: %s\n", dir);
            perror("open fial");
        }
        return;
    }

    readdir_r( dp, &entry, &entryPtr );
    while( entryPtr != NULL)
    {
        snprintf(pathbuf,2000, "%s/%s\0", dir, entry.d_name );
        printf("iteraotr:%s\n", pathbuf );

        lstat( pathbuf, &statbuf );
        if(S_ISDIR( statbuf.st_mode )) 
        {
            /* Found a directory, but ignore . and .. */
            if(strcmp(".",entry.d_name) == 0 || 
                    strcmp("..",entry.d_name) == 0)
            {

            }
            else
            {
                //printf("%d,%s\n",depth, entry->d_name);
                printdir( pathbuf, depth+1);
            }
        }
        else
        {
            //printf("%*s%s\n",depth,"",entry->d_name);
            open_file( pathbuf );
        }
        readdir_r( dp, &entry, &entryPtr );
    }
    closedir(dp);
}

//thread function   
void* iterator_dir( void* data )
{
    char* path = (char*)data;
    printf("In iterator_dir(): %s\n", path );

    printdir( path, 0 );

    return NULL;
}


pthread_t  threadID[10] = {0};

 //main function    
int main( int argc, char** argv )
{
    if( argc < 3 )
    {
        printf("Usage: %s <thread_num> <file>\n", argv[0] );
        exit(0);
    }
    if( isdigit( (char)*argv[1] ) == 0 )
    {
        printf(" Thread num is 0 - 9\n");
        exit(0);
    }

    int thread_num = atoi( argv[1] );
    char* res;

    pthread_attr_t attr;
    pthread_attr_init(&attr);
    int i = 0;
    for( i = 0; i < thread_num; ++i )
    {
        pthread_create( &threadID[i], &attr, &iterator_dir, argv[2]);
    }

    for( i = 0; i < thread_num; ++i )
    {
        pthread_join( threadID[i] , &res );
    }
}

2011-09-28 Edit:

I comment the open file operation, only keep the iterate directory part. The application still hangs. This is the output of strace:

enter code here
pid 10692] open("/home/byang//.config/menus", O_RDONLY|O_NONBLOCK|O_DIRECTORY|O_CLOEXEC <unfinished ...>
[pid 10691] write(1, "1213966080 opendir /home/byang//"..., 56) = 56

.........

[pid 10689] madvise(0x7f3c48dbc000, 8368128, MADV_DONTNEED) = 0
[pid 10689] _exit(0)                    = ?
Process 10689 detached
[pid 10688] <... futex resumed> )       = 0
[pid 10688] futex(0x7f3c47db99d0, FUTEX_WAIT, 10692, NULL <unfinished ...>

It hangs here, when I close the fanotify, it continues...

[pid 10692] <... open resumed> )        = 11
[pid 10692] getdents(11, /* 4 entries */, 32768) = 128
[pid 10692] lstat("/home/byang//.config/menus/applications.menu", {st_mode=S_IFREG|0644, st_size=233, ...}) = 0

10688 is the parent thread; 10689,10691,10692 are the child threads that iterating the directories. It seems that 10692 are waiting the reply from fanotify?

It's a bug of Kernel's fanotify. I posted a patch to Linux-Kernel: When multiple threadsiterate the same direcotry, some thread will hang. This patch let fanotify differentiate access events from different threads, prevent fanotify from merging access events from different threads.

http://marc.info/?l=linux-kernel&m=131822913806350&w=2

    -------------------------------

    diff -r -u linux-3.1-rc4_orig/fs/notify/fanotify/fanotify.c
    linux-3.1-rc4/fs/notify/fanotify/fanotify.c
    --- linux-3.1-rc4_orig/fs/notify/fanotify/fanotify.c    2011-08-29
    12:16:01.000000000 +0800
    +++ linux-3.1-rc4/fs/notify/fanotify/fanotify.c 2011-10-10
    12:28:23.276847000 +0800
    @@ -15,7 +15,8 @@

            if (old->to_tell == new->to_tell &&
                old->data_type == new->data_type &&
    -           old->tgid == new->tgid) {
    +           old->tgid == new->tgid &&
    +           old->pid == new->pid) {
                    switch (old->data_type) {
                    case (FSNOTIFY_EVENT_PATH):
                            if ((old->path.mnt == new->path.mnt) &&
    @@ -144,11 +145,19 @@
                    return PTR_ERR(notify_event);

     #ifdef CONFIG_FANOTIFY_ACCESS_PERMISSIONS
    -       if (event->mask & FAN_ALL_PERM_EVENTS) {
    -               /* if we merged we need to wait on the new event */
    -               if (notify_event)
    -                       event = notify_event;
    -               ret = fanotify_get_response_from_access(group, event);
    +       //if overflow, do not wait for response
    +       if(fsnotify_isoverflow(event))
    +       {
    +               pr_debug("fanotify overflow!\n");
    +       }
    +       else
    +       {
    +               if (event->mask & FAN_ALL_PERM_EVENTS) {
    +                       /* if we merged we need to wait on the new event */
    +                       if (notify_event)
    +                               event = notify_event;
    +                       ret = fanotify_get_response_from_access(group, event);
    +               }
            }
     #endif

    diff -r -u linux-3.1-rc4_orig/fs/notify/notification.c
    linux-3.1-rc4/fs/notify/notification.c
    --- linux-3.1-rc4_orig/fs/notify/notification.c 2011-08-29
    12:16:01.000000000 +0800
    +++ linux-3.1-rc4/fs/notify/notification.c      2011-10-10 12:27:09.331787000 +0800
    @@ -95,6 +95,7 @@
                    BUG_ON(!list_empty(&event->private_data_list));

                    kfree(event->file_name);
    +               put_pid(event->pid);
                    put_pid(event->tgid);
                    kmem_cache_free(fsnotify_event_cachep, event);
            }
    @@ -132,6 +133,14 @@
            return priv;
     }

    +bool fsnotify_isoverflow(struct fsnotify_event *event)
    +{
    +       if(event==q_overflow_event)
    +       {
    +               return true;
    +       }
    +       return false;
    +}
     /*
      * Add an event to the group notification queue.  The group can later pull this
      * event off the queue to deal with.  If the event is successfully added to the
    @@ -374,6 +383,7 @@
                            return NULL;
                    }
            }
    +       event->pid = get_pid(old_event->pid);
            event->tgid = get_pid(old_event->tgid);
            if (event->data_type == FSNOTIFY_EVENT_PATH)
                    path_get(&event->path);
    @@ -417,6 +427,7 @@
                    event->name_len = strlen(event->file_name);
            }

    +       event->pid = get_pid(task_pid(current));
            event->tgid = get_pid(task_tgid(current));
            event->sync_cookie = cookie;
            event->to_tell = to_tell;
    diff -r -u linux-3.1-rc4_orig/include/linux/fsnotify_backend.h
    linux-3.1-rc4/include/linux/fsnotify_backend.h
    --- linux-3.1-rc4_orig/include/linux/fsnotify_backend.h 2011-08-29
    12:16:01.000000000 +0800
    +++ linux-3.1-rc4/include/linux/fsnotify_backend.h      2011-10-10
    12:27:48.587369000 +0800
    @@ -238,6 +238,7 @@
            u32 sync_cookie;        /* used to corrolate events, namely inotify mv events */
            const unsigned char *file_name;
            size_t name_len;
    +       struct pid *pid;
            struct pid *tgid;

     #ifdef CONFIG_FANOTIFY_ACCESS_PERMISSIONS
    @@ -378,6 +379,8 @@
                                                            struct fsnotify_event_private_data *priv,
                                                            struct fsnotify_event *(*merge)(struct list_head *,
                                                                                            struct fsnotify_event *));
    +/*true if the event is an overflow event*/
    +extern bool fsnotify_isoverflow(struct fsnotify_event *event);
     /* true if the group notification queue is empty */
     extern bool fsnotify_notify_queue_is_empty(struct fsnotify_group *group);
     /* return, but do not dequeue the first event on the notification queue */