开发者

HTTPclient POST with problematic web site

I'm trying to retrive some data from a web site.

I wrote a java class which seems to work pretty fine with many sites but it doesn't work with this particular site, which use extensive javascript in the input fomr.

As you can see from the code I specified the input fields taking the name from the HTML source, but maybe this website doesn't accept POST request of this kind?

How can I simulate an user-interaction to retrieve the generated HTML?

package com.transport.urlRetriver;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;

import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.NameValuePair;
import org.apache.http开发者_StackOverflow中文版.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.message.BasicNameValuePair;

public class UrlRetriver {


    String stationPoller (String url, ArrayList<NameValuePair> params) {

        HttpPost postRequest;
        HttpResponse response;
        HttpEntity entity;
        String result = null;

        DefaultHttpClient httpClient = new DefaultHttpClient();


        try {

            postRequest = new HttpPost(url);

            postRequest.setEntity((HttpEntity) new UrlEncodedFormEntity(params));
            response = httpClient.execute(postRequest);

            entity = response.getEntity();

            if(entity != null){
              InputStream inputStream = entity.getContent();
              result = convertStreamToString(inputStream);
            }



        } catch (Exception e) {

            result = "We had a problem";

        } finally {

            httpClient.getConnectionManager().shutdown();

        }



        return result;

    }





    void ATMtravelPoller () {




        ArrayList<NameValuePair> params = new ArrayList<NameValuePair>(2);

        String url = "http://www.atm-mi.it/it/Pagine/default.aspx";

        params.add(new BasicNameValuePair("ctl00$SPWebPartManager1$g_afa5adbb_5b60_4e50_8da2_212a1d36e49c$txt_address_s", "Viale romagna 1"));

        params.add(new BasicNameValuePair("ctl00$SPWebPartManager1$g_afa5adbb_5b60_4e50_8da2_212a1d36e49c$txt_address_e", "Viale Toscana 20"));

        params.add(new BasicNameValuePair("sf_method", "POST"));

        String result = stationPoller(url, params);

        saveToFile(result, "/home/rachele/Documents/atm/out4.html");

    }

    static void saveToFile(String toFile, String pos){
          try{
                // Create file 
                FileWriter fstream = new FileWriter(pos);
                BufferedWriter out = new BufferedWriter(fstream);
                out.write(toFile);
                //Close the output stream
                out.close();
                }catch (Exception e){//Catch exception if any
                  System.err.println("Error: " + e.getMessage());
                }
              }

    private static String convertStreamToString(InputStream is) {
          BufferedReader reader = new BufferedReader(new InputStreamReader(is));
          StringBuilder stringBuilder = new StringBuilder();

          String line = null;
          try {
            while ((line = reader.readLine()) != null) {
              stringBuilder.append(line + "\n");
            }
          } catch (IOException e) {
            e.printStackTrace();
          } finally {
            try {
              is.close();
            } catch (IOException e) {
              e.printStackTrace();
            }
          }
          return stringBuilder.toString();
        }

}


At my point of view, there could be javascript generated field with dynamic value for preventing automated code to crawl the site. Send concrete site you want to download.

0

上一篇:

下一篇:

精彩评论

暂无评论...
验证码 换一张
取 消

最新问答

问答排行榜