Hello All,

So this script below is supposed to go to the Url listed which is a direct link to webpage which will open an excel formatted spreadsheet. Im then taking the data out of the spreadsheet and placing them within the specified variables so I can then send an email alert out on the tickets. But I get the following error after it runs for about 5 minutes: Failed to GET 'https://alacrity.secureextrasheet.com/uview?call=dg.reporting.custom.RHBgReqAlacrity&851_locale=-6.0&action=xmlapi&baseviewid=4&key=Y60f24cdc00ad0dde0c4f127cbedebbc0&reportid=8753&lob=300&resultsformat=formattedexcel': 500 Can't connect to alacrity.secureextrasheet.com:443 (connect: Connection timed out) at alacrity_defined.pl line 30. I don't have the module use Crypt::SSLeay, do I need this to connect internally to a proxy perhaps or WWW::Mechanize? Any ideas?

#!/opt/perl588/bin/perl
use strict;
use warnings;

## Required Non Standard Modules
use LWP::UserAgent;#LWP provides an objet for the application LWP. which is UserAgent for clients.  The UserAgent acts as a browser it connects to a server, sends request
#recieves repsonses and manages the recieved data.
#use Crypt::SSLeay;
use DBI;
use URI::URL;#possibly needed to be able to specify the URL for this code
use IO::Compress::Gzip qw(gzip $GzipError);
use MIME::Lite;
use Config::Std;
read_config "/var/apache2/cgi-bin/l2cgi.cfg" => my %cgi_config_hash;

#Might need the HTPP PM

$ENV{'http_proxy'} = 'http://proxy.wonu.net:8080';
$ENV{'https_proxy'} = 'http://proxy.wonu.net:8443';

## Get the URL for your flexible report from the URLs link in ConQuest Flexible Reporting
my $url = 'https://alacrity.secureextrasheet.com/uview?call=dg.reporting.custom.RHBgReqAlacrity&851_locale=-6.0&action=xmlapi&baseviewid=4&key=Y60f24cdc00ad0dde0c4f127cbedebbc0&reportid=8753&lob=300&resultsformat=formattedexcel';
#DEBUG print "Url to $url\n";  You need the URI::URL class to be able to utilize the url specification perhaps

my $ua = LWP::UserAgent->new or die "Problem with the new Useragent\n";#user agent
#DEBUG print "New LWP\n";
my $response = $ua->get($url) or die "Problem with the get $url\n";
#DEBUG print "Getting responce\n";
    
$response->is_success or die "Failed to GET '$url': ", $response->status_line;#is_success returns true when the response code is 200 - 299
#DEBUG print "About to print the html\n";

my $flexreport;
# example
#Id          #APP	#Description	
#15667354	CTRM	Headroom Breach - HR Expiry: 31-DEC-10 - Retirement Date: 29-OCT-10	
 
 if ($response->is_success)
 {
     $flexreport = $response->decoded_content;  # returns the content  with any content-Encoding undone(UTF-8) and the raw content encoded to perl's
	 #   
 }
 else 
 {
     die $response->status_line;#returns the string "<code><message>". if the message attribute is  not set then the offical name of code is sub'ed
 }

 my @information = split(/\n/, $flexreport);#split each one where there is a blank line move down to the next line and assign it to the array @information
 
 for (my $i=0;$i<=$#information;$i++) # $# references the information array to take it within the information scalar $information
 {
	my ($app, $id, ,$user, $time_open, $description) = split(/\t/, $information[$i]);#the date from the array @information is split by the blank space and tab'd to the next line for each 
   #one and assign the data to each	variable
	insert_information($app, $id, $description);#calls the subroutine insert_information with the data set in the scalar variables id, app and description which were
	#split into the scalar variables.     
	#   
 }

exit 0;

sub insert_information
{
	my $app = shift;#removes the last value of the scalar that was listed
	my $cq_id = shift;
	my $description = shift;
	
    my $dbh = DBI->connect_cached( "dbi:Sybase:TACMON_CMT", "tacmon_write", "itkgkh9" ) or die "Can't connect to database: ", $DBI::errstr, "\n";#if another call is
    #made to connect_cached with the same parameter values then the corresponding $dbh will be returned if it is valid. Th dbh is replaced with a new connection if disconnected

    $dbh->{syb_show_eed} = 1;#extended error information is is included in the string returned by $dbh-->errstr.
    $dbh->{AutoCommit}   = 0;#Turns AutoCommit off.  But it might not need to be turned off.	

    my $insert_string = "insert into dbo.TMAlerts " . "(application, environment, event_date, error_level, debug, tivoli_alert, jabber_alert, nagios_alert, email, name_space, host_name, summary) " . " VALUES (?, ?, getdate(), ?, ?, ?, ?, ?, ?, ?, ?, ?)\n";
    #$insert_string .= 'select @@identity';

    my $sth = $dbh->prepare_cached($insert_string) or return fail_sql("Can't prepare sql statement: " . $dbh->errstr );

    my $insert_result = $sth->execute( "CMT", "CONQUEST", 3, 0, 0, 1, 0, "saintclaire\@wonu.com", $cq_id, "CONQUEST", "$app - Unassigned Alacrity $cq_id" );#error level -> 3, debug -> 0 and etc
    or return fail_sql( "Can't execute SQL statement: " . $sth->errstr );
    
	my $test = $sth->errstr;

    my $ref = $sth->fetchall_arrayref;    # grabs all the data from the 1, 2 and 3 columns.
	
    my $id  = $ref->[0]->[0];#it's data is multidimensional so the first parameter is from all of the data returned and then it points to the first data. The pkey is the first
	#column in the query.

    my $blh = $dbh->prepare("select body from dbo.TMAlerts where pkey = $id")#body is an image so we would have to use the sybase connect
    or return fail_sql( "Can't prepare sql statement: " . $dbh->errstr );

    $blh->execute() or return fail_sql( "Can't execute SQL statement: " . $blh->errstr );

    while ( $blh->fetch )
	{               # don't care about the data! 
        $blh->syb_ct_data_info( 'CS_GET', 1 );
    }

    $blh->syb_ct_prepare_send();

    my $compressed_body;
    my $body = "$app - Unassigned Alacrity $cq_id\n\n$description";# this will have the app for each row from the output??
    gzip(\$body => \$compressed_body,
          Name => '', # Blank out the name of the file to save space (stored else where in table)
          '-Level' => 9)
        or fail_sql("Couldn't compress message to put into db? $GzipError");
    my $body_size = length($compressed_body);#this is needed for total_txlen variable to get the count in bytes.

    $blh->syb_ct_data_info( 'CS_SET', 1, { total_txtlen => $body_size, log_on_update => 1 } )#transfering the data in a single chunk($image, $bytes)
	      or return fail_sql( "Bad blob pointer. Null?" . $dbh->errstr() );
    #body is an image so we would have to use the blob sybase connection. And
	#set the log_on_update to TRUE by setting it to 1.
    $blh->syb_ct_send_data( $compressed_body, $body_size )
      or return fail_sql( "Can't execute blob update: " . $sth->errstr() . "----");

    $blh->syb_ct_finish_send();#commits the process

    $dbh->commit()    # Commit tran once the unlink has completed so re-run never hits a dupe row as we turned off Auto Commit on line 71
	# Commit tran once the unlink has completed so re-run never hits a dupe row.
      or return fail_sql( "Commit failed!" . $dbh->errstr );
    return 1;

    $dbh->disconnect
        or print "Disconnect Error:  $dbh->errstr()";
         warn "Failed to disconnect: ", $dbh->errstr(), "\n" if $dbh->errstr();

}

Edited 5 Years Ago by choosenalpha: n/a

I don't show the login process for your code. Include the login credentials, add the "cookie_jar" and include the 'https' keyword in the "protocols_allowed" options to the constructor. That may be helpuful for you.

The login credentials are on line 67 for the DB connections. For the website it is within the proxy(intranet). Can you elaborate on adding the "cookie_jar" and including the 'https' keyword in the "protocols_allowed" and also to what constructor? Is this within another module and am I missing any modules that could make this simplier or that could make this work?

Thanks and Regards,

I don't show the login process for your code. Include the login credentials, add the "cookie_jar" and include the 'https' keyword in the "protocols_allowed" options to the constructor. That may be helpuful for you.

Yes, I noticed the DB credentials. I meant the webpage credentials.
You must pass the webpage login credentials for the metioned URL.
Refer the 'LWP::UserAgent' documentation in CPAN.

Sorry for the late reply not feeling well the past couple of days. Ok so for this script I cut out the other code and just worked on the portion where it is connecting. So I was able to troubleshoot and connect but now I am getting a 400 Bad Request error please see my code below. And here is the error: > perl conquest_download.pl
New LWP
Getting response
Failed to GET 'https://alacrity.secureextrasheet.com/uview?call=dg.reporting.custom.RHBgReqAlacrity&851_locale=-5.0&action=xmlapi&baseviewid=4&key=Y60f24cdc00ad0dde0c4f127cbedebbc0&reportid=10934&lob=300&resultsformat=txtfile': 400 Bad Request at conquest_download.pl line 38.

#!/opt/perl588/bin/perl
use strict;
use warnings;

## Required Non Standard Modules
use LWP::UserAgent;#LWP provides an object for the application LWP. which is UserAgent for clients.  The UserAgent acts as a browser it connects to a server, sends request
#recieves repsonses and manages the recieved data.
#use Crypt::SSLeay;
use LWP::Simple;
use DBI;
use URI::URL;#possibly needed to be able to specify the URL for this code
use IO::Compress::Gzip qw(gzip $GzipError);
use MIME::Lite;
use Config::Std;
use HTTP::Request;

read_config "/var/apache2/cgi-bin/l2cgi.cfg" => my %cgi_config_hash;

#Might need the HTPP PM

$ENV{'http_proxy'} = 'http://proxy.wonu.net:8080';  #this might be needed to set the proxy need to verify
$ENV{'https_proxy'} = 'http://proxy.wonu.net:8443'; #need to double check if this is needed or not

#--------TAKE DATA FROM URL AND SAVE DOWN TO LOCAL DIRECTORY WITHIN CGI-BIN

## Get the URL for your flexible report from the URLs link in ConQuest Flexible Reporting
my $url = 'https://alacrity.secureextrasheet.com/uview?call=dg.reporting.custom.RHBgReqAlacrity&851_locale=-5.0&action=xmlapi&baseviewid=4&key=Y60f24cdc00ad0dde0c4f127cbedebbc0&reportid=10934&lob=300&resultsformat=txtfile';
#DEBUG print "Url to $url\n";  You need the URI::URL class to be able to utilize the url specification perhaps

#$browser-> timeout(15);
my $ua = LWP::UserAgent->new or die "Problem with the new Useragent\n";#user agent
$ua->proxy(['https'],$ENV{'https_proxy'});
print "New LWP\n";
#my $response = HTTP::Request->get('https://alacrity.secureextrasheet.com/uview?call=dg.reporting.custom.RHBgReqAlacrity&851_locale=-5.0&action=xmlapi&baseviewid=4&key=Y60f24cdc00ad0dde0c4f127cbedebbc0&reportid=10934&lob=300&resultsformat=txtfile') or die "There is a freaking problem getting $url\n";
my $response = $ua->get($url) or die "Problem with the get $url\n";
print "Getting response\n";

$response->is_success or die "Failed to GET '$url': ", $response->status_line;#is_success returns true when the response code is 200 - 299
print "Able to grab the URL we are moving forward baby!";#debug

Yes, I noticed the DB credentials. I meant the webpage credentials.
You must pass the webpage login credentials for the metioned URL.
Refer the 'LWP::UserAgent' documentation in CPAN.

1) For here no need the use the proxy.
2) Still the login process missing in the code.
3) Try the below code...I hope you may get some another way.

use WWW::Mechanize;

## Declare your the url
$url = 'https://alacrity.secureextrasheet.com';
$m = WWW::Mechanize->new();
$m->get($url);

## Declare the login credentials
$login = 'Your Login Name';
$password = 'Your Password Here';

$m->submit_form(
        form_number => 1,  # Analyze the page source & modify the code's
        fields      => { username_field => $login , username_field => $password },
    );
    die unless ($m->success);

if ( $m->success )
{
	print "\nLogin Successfully";
}
else
{
	print "\nLogin Failed";
	exit;
}

$url = 'https://alacrity.secureextrasheet.com/uview?call=dg.reporting.custom.RHBgReqAlacrity&851_locale=-5.0&action=xmlapi&baseviewid=4&key=Y60f24cdc00ad0dde0c4f127cbedebbc0&reportid=10934&lob=300&resultsformat=txtfile';
$m->get($url);
$content = $m->content;
print "\nPage Conent : $content";

Ok thanks im trying this method and now getting this error message below. I believe I have an updated Perl Module package; I'll tinker around and see what I have and see if I can fix it.

Can't locate object method "success" via package "HTTP::Headers"

1) For here no need the use the proxy.
2) Still the login process missing in the code.
3) Try the below code...I hope you may get some another way.

use WWW::Mechanize;

## Declare your the url
$url = 'https://alacrity.secureextrasheet.com';
$m = WWW::Mechanize->new();
$m->get($url);

## Declare the login credentials
$login = 'Your Login Name';
$password = 'Your Password Here';

$m->submit_form(
        form_number => 1,  # Analyze the page source & modify the code's
        fields      => { username_field => $login , username_field => $password },
    );
    die unless ($m->success);

if ( $m->success )
{
	print "\nLogin Successfully";
}
else
{
	print "\nLogin Failed";
	exit;
}

$url = 'https://alacrity.secureextrasheet.com/uview?call=dg.reporting.custom.RHBgReqAlacrity&851_locale=-5.0&action=xmlapi&baseviewid=4&key=Y60f24cdc00ad0dde0c4f127cbedebbc0&reportid=10934&lob=300&resultsformat=txtfile';
$m->get($url);
$content = $m->content;
print "\nPage Conent : $content";

Got it working thanks k_manimuthu very much! Got it working here is my login portion now I am working on the other piece of the code for the formatting :)

> perl new_conquest2.pl
The login page is now opened!....
Login is successful with login credentials

#!/opt/perl588/bin/perl
use WWW::Mechanize;
use strict;
use HTTP::Headers;
##Url to login to SSO page

my $mech = WWW::Mechanize->new();
##SSO login credentials
my $login ='mfuouo';
my $password ='ajfkd3';

my $login_url = 'http://alacrity.wonu.com:8084/alacrity/Secure/AlacrityLogin.jsp';
my $result=$mech->get($login_url);
die "GET failed\n" unless ($result->content);
print "The login page is now opened!....\n";

$result = $mech->submit_form(
		form_name => 'Login', #Analyze the page source & modify the code's
		fields => {standard_id => $login, password => $password}
		);
		die "GET failed\n" unless ($result->content);
		print "Login is successful with login credentials\n";

Ok thanks im trying this method and now getting this error message below. I believe I have an updated Perl Module package; I'll tinker around and see what I have and see if I can fix it.

Can't locate object method "success" via package "HTTP::Headers"

Edited 5 Years Ago by choosenalpha: n/a

Hi Mani!

Yes that's what I thought too..but then I got this error below..it seems to have timed out. I'm thinking if it is using the same login credential and it isn't using the a different instance of the Mechanize function then it be able to pick up the report right? But it is timing out as seen below and here is the error. Am I missing something possibly?

perl new_conquest2.pl
The login page is now opened!....
Login is successful with login credentials
Ok we got the report url....
It pulled the content from the report

Page Content : 500 Can't connect to alacrity.secureextrasheet.com:443 (connect: Connection timed out)

#!/opt/perl588/bin/perl
use WWW::Mechanize;
use strict;
use HTTP::Headers;
##Url to login to SSO page

my $mech = WWW::Mechanize->new();
##SSO login credentials
my $login ='mfuouo';
my $password ='ajfkd3';

my $login_url = 'http://alacrity.wonu.com:8084/alacrity/Secure/AlacrityLogin.jsp';
my $report_url = 'https://alacrity.secureextrasheet.com/uview?call=dg.reporting.custom.RHBgReqAlacrity&851_locale=-5.0&action=xmlapi&baseviewid=4&key=Y60f24cdc00ad0dde0c4f127cbedebbc0&reportid=10934&lob=300&resultsformat=txtfile';
my $result=$mech->get($login_url);
die "GET failed\n" unless ($result->content);
print "The login page is now opened!....\n";

$result = $mech->submit_form(
		form_name => 'Login', #Analyze the page source & modify the code's
		fields => {standard_id => $login, password => $password}
		);
		die "GET failed\n" unless ($result->content);
		print "Login is successful with login credentials\n";
##Grabbing the flexreport once logged in successfully

my $mech_flexreport=$mech->get($report_url);
print "Ok we got the report url....\n";
die "GET failed\n" unless ($mech_flexreport->content);
print "It pulled the content from the report\n";
my $content = $mech_flexreport->content;
print "\nPage Content : $content";
$url = 'https://alacrity.secureextrasheet.com/uview?call=dg.reporting.custom.RHBgReqAlacrity&851_locale=-5.0&action=xmlapi&baseviewid=4&key=Y60f24cdc00ad0dde0c4f127cbedebbc0&reportid=10934&lob=300&resultsformat=txtfile';
$m->get($url);
$content = $mech->content;
print "\nPage Conent : $content";

Well, If you able to get the content it could be easy to do the automation.

die "GET failed\n" unless ($result->content);

Insist of the above line, check the login status by using of 'success' function.

if ( $mech->success )
{
print "\nLogin Successfully";
}
else
{
print "\nLogin Failed";
exit;
}

The default timeout() value is 180 seconds. I am not sure about this.
You may set the timeout function "$mech->timeout( 'Mentioned your time' );" and try again.

For more details.
http://search.cpan.org/dist/WWW-Mechanize/lib/WWW/Mechanize/FAQ.pod#My_Mech_program_gets_these_500_errors.

Edited 5 Years Ago by k_manimuthu: n/a

This article has been dead for over six months. Start a new discussion instead.