Web Automation

[ToRtUgOxX]

I m searching for a library for filling forms, and click buttons. in C++ , linux or windows.
Thanks.

[Ancient Dragon]

try c# or c++/CLR

[Ancient Dragon]

Here is an article you might want to study

[ToRtUgOxX]

thank you ancient dragon, i ll take a look.

[NicAx64]

or you can just open the socket.
Use HTTP protocol,it's easy.

just socket() connect() read()/write()
it's simple than you ever think.
And the good news is inside linux these libraries are already there.
(these are system calls in the POSIX interface,so every *nix got these).

and if you decided to do this with linux Posix system calls I could
help you further.

to access to the documentation (for example socket() function).
Type
#man 3 socket
#man 3 connect
etc.

POSIX API is a the best and suitable method to do this if your under linux.

[ToRtUgOxX]

Can someone post an example for connecting google, fill the search box, and click submit button, i could do that using perl, and the samie library, but i want it in C, thanks.
i got nowhere.

[gashtio]

Here's some code that issues a GET request and writes the response to file. I tested it on Windows only, but it should work on Linux, too.

It doesn't exactly "fill the search box, and click submit button", but should be enough to get you started.

#ifdef WIN32
	#define _CRT_SECURE_NO_WARNINGS
	#include <stdio.h>
	#include <string.h>
	#include <Ws2tcpip.h>
	#define LASTERROR WSAGetLastError()
	#define close_socket(sockfd) closesocket(sockfd)
	#define cleanup(sockfd) { closesocket(sockfd); WSACleanup(); }
	#pragma comment( lib, "WS2_32.lib" )
#else
	#include <unistd.h>
	#include <stdio.h>
	#include <string.h>
	#include <sys/types.h>
	#include <sys/socket.h>
	#include <arpa/inet.h>
	#include <netdb.h>
	#include <errno.h>
	typedef int SOCKET;
	#define INVALID_SOCKET -1
	#define SOCKET_ERROR -1
	#define LASTERROR errno
	#define close_socket(sockfd) close(sockfd)
	#define cleanup(sockfd) close(sockfd)
#endif

#define OUTPUTFILE "response.html"
#define MAXDATASIZE 8000 // max number of bytes we can get at once

//--------------------------------------------------------------------------------------

static void * get_in_addr( struct sockaddr * sa )
{
	if ( sa->sa_family == AF_INET ) // IPv4 address
		return &(((struct sockaddr_in *)sa)->sin_addr);
	else // IPv6 address
		return &(((struct sockaddr_in6 *)sa)->sin6_addr);
}

//--------------------------------------------------------------------------------------

int main()
{
	FILE * hOutput;
	SOCKET sockfd;
	char Buffer[MAXDATASIZE];
	struct addrinfo hints, *pResults, *p;
	int ret;

	char szDotDecimalIp[INET6_ADDRSTRLEN];
	const char * szQuery = "/search?q=web+automation+c%2B%2B";
	const char * szHost = "www.google.com";
	char szHttpRequest[1024];
	
#ifdef WIN32
	const int op_timeout_ms = 2000; // Timeout for send/recv
	WSADATA wsa;
	WSAStartup( MAKEWORD(2,2), &wsa );
#endif

	// ---- Get the IPs of the host
	memset( &hints, 0, sizeof hints );
	hints.ai_family	  = AF_UNSPEC;
	hints.ai_socktype = SOCK_STREAM; // TCP socket
	if ( (ret = getaddrinfo( szHost, "http", &hints, &pResults )) != 0 )
	{
#if defined(WIN32) & defined(UNICODE)
		fwprintf( stderr, L"getaddrinfo: %s\n", gai_strerror(ret) );
#else
		fprintf( stderr, "getaddrinfo: %s\n", gai_strerror(ret) );
#endif
		return -1;
	}

	// ---- Loop through all the results and connect to the first we can
	for( p = pResults; p != NULL; p = p->ai_next )
	{
		if ( (sockfd = socket( p->ai_family, p->ai_socktype, p->ai_protocol )) != INVALID_SOCKET )
		{
			if ( connect( sockfd, p->ai_addr, p->ai_addrlen ) != SOCKET_ERROR )
			{
				// Connection successful - break out of the loop
				break;
			}
			else
			{
				close_socket(sockfd);
			}
		}
	}

	if ( p == NULL )
	{
		fprintf( stderr, "Failed establishing a connection\n" );
		return -2;
	}

	// ---- Print out the address we're connecting to
	inet_ntop( p->ai_family, get_in_addr((struct sockaddr *)p->ai_addr), szDotDecimalIp, sizeof szDotDecimalIp );
	printf( "Connected to %s\n", szDotDecimalIp );
	freeaddrinfo( pResults ); // No longer needed
	

#ifdef WIN32
	setsockopt( sockfd, SOL_SOCKET, SO_SNDTIMEO, (const char *)&op_timeout_ms, sizeof op_timeout_ms );
	setsockopt( sockfd, SOL_SOCKET, SO_RCVTIMEO, (const char *)&op_timeout_ms, sizeof op_timeout_ms );
#endif


	// ---- Construct a GET request
        ret = 0;
        ret += sprintf( szHttpRequest + ret, "GET %s HTTP/1.1\r\n", szQuery );
        ret += sprintf( szHttpRequest + ret, "Host: %s\r\n", szHost );
        ret += sprintf( szHttpRequest + ret, "Connection: close\r\n" );
        ret += sprintf( szHttpRequest + ret, "User-Agent: Mozilla/5.0 (Windows; U; Windows NT 6.1;"
            "en-GB; rv:1.9.2.3) Gecko/20100401 Firefox/3.6.3\r\n" );
        ret += sprintf( szHttpRequest + ret, "Accept-Language: en-gb,en;q=0.5\r\n" );
        // If you uncomment the next line, you'll have to decompress the response body (e.g. with a library such as zlib).
        //ret += sprintf( szHttpRequest + ret, "Accept-Encoding: gzip\r\n" );
        ret += sprintf( szHttpRequest + ret, "Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7\r\n" );
        ret += sprintf( szHttpRequest + ret, "Cache-Control: no\r\n" );
        ret += sprintf( szHttpRequest + ret, "\r\n" );
	
	// ---- Send request
	if ( (ret = send( sockfd, szHttpRequest, strlen(szHttpRequest), 0 )) == SOCKET_ERROR )
	{
		fprintf( stderr, "send() failed with error code #%d\n", LASTERROR );
		cleanup( sockfd );
		return -3;
	}

	
	// ---- Get response and write it to file
	hOutput = fopen( OUTPUTFILE, "w" );

	while ( (ret = recv( sockfd, Buffer, MAXDATASIZE-1, 0 )) > 0 )
	{
		fwrite( Buffer, 1, ret, hOutput );
		//printf( "Writing %d bytes\n", ret );
	}

	if ( ret == SOCKET_ERROR )
	{
		fprintf( stderr, "recv() failed with error code #%d\n", LASTERROR );
	}

	fclose( hOutput );
	cleanup( sockfd );
	
	return 0;
}

However, when using sockets directly for HTTP you don't get much help from the API - e.g. you have to deal with gzip encoding, chunked transfer, searching for a blank line to separate the response header from the body and so on yourself (none of these are implemented in the code above). To avoid such mundane tasks, I'd suggest you try the cURL library, which is very easy to use and has tons of features. Here's the same code using cURL, complete with automatic header parsing:

#define _CRT_SECURE_NO_WARNINGS
#include <stdio.h>
#include "curl/curl.h"
#define OUTPUTFILE "response.html"

//--------------------------------------------------------------------------------------

int main()
{
	FILE * hOutput;
	CURL * hCurl;
	CURLcode ret;
	char szError[CURL_ERROR_SIZE] = { 0 };

	// ---- Init cURL
	hCurl = curl_easy_init();
	if ( ! hCurl )
	{
		fprintf( stderr, "Initializing cURL failed!\n" );
		return -1;
	}

	// ---- Open output file
	hOutput = fopen( OUTPUTFILE, "w" );
	if ( ! hOutput )
	{
		fprintf( stderr, "Opening %s failed!\n", OUTPUTFILE );
		curl_easy_cleanup( hCurl );
		return -2;
	}

	// ---- Set options and perform file transfer
	curl_easy_setopt( hCurl, CURLOPT_WRITEDATA, hOutput ); // Redirect output

	curl_easy_setopt( hCurl, CURLOPT_ERRORBUFFER, szError ); // Set error buffer

	curl_easy_setopt( hCurl, CURLOPT_HTTPGET, 1L );
	curl_easy_setopt( hCurl, CURLOPT_URL, "www.google.com/search?q=web+automation+c%2B%2B" );

	ret = curl_easy_perform( hCurl );
	if ( ret )
	{
		fprintf( stderr, "Error #%d: %s\n", ret, szError );
	}

	// ---- Cleanup
	fclose( hOutput );
	curl_easy_cleanup( hCurl );

	return 0;
}

You can download the library from curl.haxx.se, and check out more examples at curl.haxx.se/libcurl/c/example.html.