verify digital signature using public key in openssl

I have signed a data in windows using wincrypt cryptoapi (PKCS_7_ASN_ENCODING | X509_ASN_ENCODING) and in linux, I have x509 certificate and the signed message which i have to verify

Code in windows to sign :
hStoreHandle = CertOpenStore(
    CERT_STORE_PROV_SYSTEM,
    0,
    NULL,
    CERT_SYSTEM_STORE_CURRENT_USER,
    CERT_PERSONAL_STORE_NAME
);
CheckError((BOOL)hStoreHandle, L"CertOpenStore....................... ");

// Get signer's certificate with access to private key.
do {
    // Get a certificate that matches the search criteria
    pSignerCert = CertFindCertificateInStore(
        hStoreHandle,
        MY_TYPE,
        0,
        CERT_FIND_SUBJECT_STR,
        SignerName,
        pSignerCert
    );
    CheckError((BOOL)pSignerCert, L"CertFindCertificateInStore.......... ");

    // Get the CSP, and check if we can sign with the private key           
    bResult = CryptAcquireCertificatePrivateKey(
        pSignerCert,
        0,
        NULL,
        &hCryptProv,
        &dwKeySpec,
        NULL
    );
    CheckError(bResult, L"CryptAcquireCertificatePrivateKey... ");

} while ((dwKeySpec & AT_SIGNATURE) != AT_SIGNATURE);

// Create the hash object.
bResult = CryptCreateHash(
    hCryptProv, 
    CALG_MD5, 
    0, 
    0, 
    &hHash
);
CheckError(bResult, L"CryptCreateHash..................... ");

// Open the file with the content to be signed 
hDataFile = CreateFileW(DataFileName,
    GENERIC_READ,
    FILE_SHARE_READ,
    NULL,
    OPEN_EXISTING,
    FILE_FLAG_SEQUENTIAL_SCAN,
    NULL
);
CheckError((hDataFile != INVALID_HANDLE_VALUE), L"CreateFile.......................... ");

// Compute the cryptographic hash of the data.
while (bResult = ReadFile(hDataFile, rgbFile, BUFSIZE, &cbRead, NULL))
{
    if (cbRead == 0)
    {
        break;
    }
    CheckError(bResult, L"ReadFile............................ ");

    bResult = CryptHashData(
        hHash, 
  开发者_如何学Python      rgbFile, 
        cbRead, 
        0
    );
    CheckError(bResult, L"CryptHashData....................... ");

}
CheckError(bResult, L"ReadFile............................ ");

// Sign the hash object
dwSigLen = 0;
bResult = CryptSignHash(
    hHash, 
    AT_SIGNATURE, 
    NULL, 
    0, 
    NULL, 
    &dwSigLen
);
CheckError(bResult, L"CryptSignHash....................... ");

pbSignature = (BYTE *)malloc(dwSigLen);
CheckError((BOOL)pbSignature, L"malloc.............................. ");

bResult = CryptSignHash(
    hHash, 
    AT_SIGNATURE, 
    NULL, 
    0, 
    pbSignature, 
    &dwSigLen
);
CheckError(bResult, L"CryptSignHash....................... ");

// Create a file to save the signature
hSignatureFile = CreateFileW(
    SignatureFileName,
    GENERIC_WRITE,
    0,
    NULL,
    CREATE_ALWAYS,
    FILE_ATTRIBUTE_NORMAL,
    NULL
);
CheckError((hSignatureFile != INVALID_HANDLE_VALUE), L"CreateFile.......................... ");

// Write the signature to the file
bResult = WriteFile(
    hSignatureFile, 
    (LPCVOID)pbSignature, 
    dwSigLen, 
    &lpNumberOfBytesWritten, 
    NULL
);
CheckError(bResult, L"WriteFile........................... ");



In openssl i tried:

 openssl rsautl -verify -inkey pubkey.pem -keyform PEM -pubin -in signedmessage   

it is throwing error::   
RSA operation error  
4296:error:0406706C:rsa routines:RSA_EAY_PUBLIC_DECRYPT:data greater than modlen:fips_rsa_eay.c:709:  
and this error if the signedmessage is hashed
RSA operation error
4432:error:0407006A:rsa routines:RSA_padding_check_PKCS1_type_1:block type is not     01:rsa_pk1.c:100:
4432:error:04067072:rsa routines:RSA_EAY_PUBLIC_DECRYPT:padding check     failed:fips_rsa_eay.c:748:


i also tried :  
   openssl dgst -verify pubkey.pem -signature signedmessage
but program goes into infinite loop

I also find one command:
 int PKCS7_verify(PKCS7 *p7, STACK_OF(X509) *certs, X509_STORE *store, BIO *indata, BIO *out, int flags);
     but it require too many argument of which i am not aware of.e.g there is no x509_store, crlfile used in this  

. can any one tell me how to verify the signed message

I get x509 pem certificate and signedmessage as input in linux which i have to verify

After some example by mail, we got to the following recipe

setup: we have a x509 certificate cert.p7b to start with, a file message.txt, a Windows produced signed.dat, and using sha1 for definiteness.

openssl pkcs7 -inform DER -outform PEM -in cert.p7b -out cert.pem -print_certs

openssl x509 -in cert.pem -noout -pubkey > pubkey.pem

(this need only be done once for a certificate, to get a public key in PEM format) then reverse signed.dat bytewise to signed.dat.rev (using a simple C program, or output the bytes differently on Windows, in alternative form) and finally

openssl dgst -sha1 -verify pubkey.pem -signature signed.dat.rev message.txt

The main problem was the reverse byte order on Windows (which I have seen before)