Spike in "500 - internal server errors" from web services running on IIS7, Windows Server 2008

We have a bunch of websites running on a NLB of 3 web servers. One of the sites is a set of web services which is used to power a news application for an iPad.

On Monday the client called up to say they were seeing some errors on the App. I jumped on the server and noticed a lot of 500s in the IIS logs. I then wrote a script to call the webserverice and managed to reproduce the 500 through fiddler but the response was useless.

The web service gets around 200,000 hits a day. Typically we get less than 10 "500 internal server errors per day". On Sunday however these spiked to 2000 (around 1% of all requests) and have been at that level ever since.

We have had no code deploys since mid June. Our server/network infrastructure has not changed in anyway and I can't explain to the client what the issue is.

I have enabled Troubleshooting Failed Requests Using Tracing in IIS 7 as suggested in this post but the error message is meaningless to me. I have Googled extensively but no dice, trace details below:

114. -MODULE_SET_RESPONSE_ERROR_STATUS 
ModuleName ManagedPipelineHandler 
Notification 128 
HttpStatus 500 
HttpReason Internal Server Error 
HttpSubStatus 0 
ErrorCode 0 
ConfigExceptionInfo  
Notification EXECUTE_REQUEST_HANDLER 
ErrorCode The operation completed successfully. (0x0) 

The web config is here (with irrelevant parts removed)

<?xml version="1.0" encoding="UTF-8"?>

<configSections>
    <sectionGroup name="system.web.extensions" type="System.Web.Confi开发者_StackOverflowguration.SystemWebExtensionsSectionGroup, System.Web.Extensions, Version=3.5.0.0, Culture=neutral, PublicKeyToken=31BF3856AD364E35">
        <sectionGroup name="scripting" type="System.Web.Configuration.ScriptingSectionGroup, System.Web.Extensions, Version=3.5.0.0, Culture=neutral, PublicKeyToken=31BF3856AD364E35">
            <section name="scriptResourceHandler" type="System.Web.Configuration.ScriptingScriptResourceHandlerSection, System.Web.Extensions, Version=3.5.0.0, Culture=neutral, PublicKeyToken=31BF3856AD364E35" requirePermission="false" allowDefinition="MachineToApplication" />
            <sectionGroup name="webServices" type="System.Web.Configuration.ScriptingWebServicesSectionGroup, System.Web.Extensions, Version=3.5.0.0, Culture=neutral, PublicKeyToken=31BF3856AD364E35">
                <section name="jsonSerialization" type="System.Web.Configuration.ScriptingJsonSerializationSection, System.Web.Extensions, Version=3.5.0.0, Culture=neutral, PublicKeyToken=31BF3856AD364E35" requirePermission="false" allowDefinition="Everywhere" />
                <section name="profileService" type="System.Web.Configuration.ScriptingProfileServiceSection, System.Web.Extensions, Version=3.5.0.0, Culture=neutral, PublicKeyToken=31BF3856AD364E35" requirePermission="false" allowDefinition="MachineToApplication" />
                <section name="authenticationService" type="System.Web.Configuration.ScriptingAuthenticationServiceSection, System.Web.Extensions, Version=3.5.0.0, Culture=neutral, PublicKeyToken=31BF3856AD364E35" requirePermission="false" allowDefinition="MachineToApplication" />
                <section name="roleService" type="System.Web.Configuration.ScriptingRoleServiceSection, System.Web.Extensions, Version=3.5.0.0, Culture=neutral, PublicKeyToken=31BF3856AD364E35" requirePermission="false" allowDefinition="MachineToApplication" />
            </sectionGroup>
        </sectionGroup>
    </sectionGroup>

</configSections>

<appSettings />
<connectionStrings />

<system.web>
    <webServices>
        <protocols>
            <add name="HttpGet" />
            <add name="HttpPost" />
        </protocols>
    </webServices>
    <compilation debug="false">
        <assemblies>
            <add assembly="System.Core, Version=3.5.0.0, Culture=neutral, PublicKeyToken=B77A5C561934E089" />
            <add assembly="System.Data.DataSetExtensions, Version=3.5.0.0, Culture=neutral, PublicKeyToken=B77A5C561934E089" />
            <add assembly="System.Web.Extensions, Version=3.5.0.0, Culture=neutral, PublicKeyToken=31BF3856AD364E35" />
            <add assembly="System.Xml.Linq, Version=3.5.0.0, Culture=neutral, PublicKeyToken=B77A5C561934E089" />
        </assemblies>
    </compilation>
    <pages>
        <controls>
            <add tagPrefix="asp" namespace="System.Web.UI" assembly="System.Web.Extensions, Version=3.5.0.0, Culture=neutral, PublicKeyToken=31BF3856AD364E35" />
            <add tagPrefix="asp" namespace="System.Web.UI.WebControls" assembly="System.Web.Extensions, Version=3.5.0.0, Culture=neutral, PublicKeyToken=31BF3856AD364E35" />
        </controls>
    </pages>
    <httpHandlers>
        <remove verb="*" path="*.asmx" />
        <add verb="*" path="*.asmx" validate="false" type="System.Web.Script.Services.ScriptHandlerFactory, System.Web.Extensions, Version=3.5.0.0, Culture=neutral, PublicKeyToken=31BF3856AD364E35" />
        <add verb="*" path="*_AppService.axd" validate="false" type="System.Web.Script.Services.ScriptHandlerFactory, System.Web.Extensions, Version=3.5.0.0, Culture=neutral, PublicKeyToken=31BF3856AD364E35" />
        <add verb="GET,HEAD" path="ScriptResource.axd" type="System.Web.Handlers.ScriptResourceHandler, System.Web.Extensions, Version=3.5.0.0, Culture=neutral, PublicKeyToken=31BF3856AD364E35" validate="false" />
    </httpHandlers>
    <httpModules>
        <add name="ScriptModule" type="System.Web.Handlers.ScriptModule, System.Web.Extensions, Version=3.5.0.0, Culture=neutral, PublicKeyToken=31BF3856AD364E35" />
    </httpModules>
</system.web>
<system.codedom>
    <compilers>
        <compiler language="c#;cs;csharp" extension=".cs" warningLevel="4" type="Microsoft.CSharp.CSharpCodeProvider, System, Version=2.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089">
            <providerOption name="CompilerVersion" value="v3.5" />
            <providerOption name="WarnAsError" value="false" />
        </compiler>
    </compilers>
</system.codedom>

<!-- 
    The system.webServer section is required for running ASP.NET AJAX under Internet
    Information Services 7.0.  It is not necessary for previous version of IIS.
-->
<system.webServer>
    <validation validateIntegratedModeConfiguration="false" />
    <modules>
        <remove name="ScriptModule" />
        <add name="ScriptModule" preCondition="managedHandler" type="System.Web.Handlers.ScriptModule, System.Web.Extensions, Version=3.5.0.0, Culture=neutral, PublicKeyToken=31BF3856AD364E35" />
    </modules>
    <handlers>
        <remove name="WebServiceHandlerFactory-Integrated" />
        <remove name="ScriptHandlerFactory" />
        <remove name="ScriptHandlerFactoryAppServices" />
        <remove name="ScriptResource" />
        <add name="ScriptHandlerFactory" verb="*" path="*.asmx" preCondition="integratedMode" type="System.Web.Script.Services.ScriptHandlerFactory, System.Web.Extensions, Version=3.5.0.0, Culture=neutral, PublicKeyToken=31BF3856AD364E35" />
        <add name="ScriptHandlerFactoryAppServices" verb="*" path="*_AppService.axd" preCondition="integratedMode" type="System.Web.Script.Services.ScriptHandlerFactory, System.Web.Extensions, Version=3.5.0.0, Culture=neutral, PublicKeyToken=31BF3856AD364E35" />
        <add name="ScriptResource" preCondition="integratedMode" verb="GET,HEAD" path="ScriptResource.axd" type="System.Web.Handlers.ScriptResourceHandler, System.Web.Extensions, Version=3.5.0.0, Culture=neutral, PublicKeyToken=31BF3856AD364E35" />
    </handlers>
    <tracing>
        <traceFailedRequests>
            <add path="*.asmx">
                <traceAreas>
                    <add provider="ASP" verbosity="Verbose" />
                    <add provider="ASPNET" areas="Infrastructure,Module,Page,AppServices" verbosity="Verbose" />
                    <add provider="ISAPI Extension" verbosity="Verbose" />
                    <add provider="WWW Server" areas="Rewrite,Authentication,Security,Filter,StaticFile,CGI,Compression,Cache,RequestNotifications,Module" verbosity="Verbose" />
                </traceAreas>
                <failureDefinitions timeTaken="00:00:00" statusCodes="500" />
            </add>
        </traceFailedRequests>
    </tracing>
</system.webServer>

<runtime>
    <assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1" appliesTo="v2.0.50727">
        <dependentAssembly>
            <assemblyIdentity name="System.Web.Extensions" publicKeyToken="31bf3856ad364e35" />
            <bindingRedirect oldVersion="1.0.0.0-1.1.0.0" newVersion="3.5.0.0" />
        </dependentAssembly>
        <dependentAssembly>
            <assemblyIdentity name="System.Web.Extensions.Design" publicKeyToken="31bf3856ad364e35" />
            <bindingRedirect oldVersion="1.0.0.0-1.1.0.0" newVersion="3.5.0.0" />
        </dependentAssembly>
    </assemblyBinding>
</runtime>