Integrating Selenium And CruiseControl.Net - Selenium

This article will provide more detail about integrating Selenium into your ContinuousIntegration engine. The example will be based on CruiseControl.Net and Nant but can easily be ported to regular CruiseControl and Ant. I assume that you are familiar with Continuous Integration, Nant and Selenium.

To achieve Continuous Integration (CI) with selenium you need to achieve the following steps

Selenium needs to run automatically, and save its results to file
Nant script needs to call selenium
Nant needs to wait for the results
Nant needs to react to those results (i.e. should we fail on errors)

To have Selenium run automatically, simply add auto=true to the command line. To record the results of a test run, also add resultsUrl=tests\HandleResults.php.

For example:

http://localhost/Selenium/TestRunner.html?auto=true&resultsUrl=tests\HandleResults.php

The content of HandleResults.php is outlined below:

<?

if (!isset($HTTP_POST_VARS["result"]))
{
  ?>No Results<?
  return;
}

$result = $HTTP_POST_VARS["result"];
$totalTime =$HTTP_POST_VARS["totalTime"];
$numberOfSuccesses = $HTTP_POST_VARS["numTestPasses"];
$numberOfFailures = $HTTP_POST_VARS["numTestFailures"];

$numberOfCommandSuccesses = $HTTP_POST_VARS["numCommandPasses"];
$numberOfCommandFailures = $HTTP_POST_VARS["numCommandFailures"];
$numberOfCommandErrors = $HTTP_POST_VARS["numCommandErrors"];


$xmlText = "<selenium " .
  "result=\"$result\" " .
  "totalTime=\"$totalTime\" " .
  "successes=\"$numberOfCommandSuccesses\" " .
  "failures=\"$numberOfCommandFailures\" " .
  "errors=\"$numberOfCommandErrors\" ".
  "/>";

$handle = fopen("TestSummary.xml", "w");
$writer = fwrite($handle, $xmlText);
fclose($handle);

?>

Result: <?= $result ?><br>
Total Time: <?= $totalTime ?><br>
# Passes: <?= $numberOfSuccesses ?><br>
# Failures: <?= $numberOfFailures ?><br>
# Command Passes: <?= $numberOfCommandSuccesses ?><br>
# Command Failures: <?= $numberOfCommandFailures ?><br>
# Command Errors: <?= $numberOfCommandErrors ?><br>

<br><br>
<?= htmlspecialchars($xmlText) ?>

The above task will simply write the results of the test run to file. The resulting XML looks simliar to:

<selenium result="failed" totalTime="57" successes="43" failures="1" errors="0" />

Now that we can record the results of a Selenium test run to file, we need Nant to be able to call that URL above, and react to the results as necessary. The task would look similar to:

<selenium
  browserExe="C:\Program Files\Internet Explorer\iexplore.exe"    
  testSuiteUrl="http://localhost/Selenium/TestRunner.html?auto=true&amp;resultsUrl=tests\HandleResults.php"
  maximumWaitTimeInSeconds="100"
  resultsFile="Selenium\tests\TestSummary.xml" 
  failonerror="false"
/>

Please refer to the resources at the end of this article for the source code to the selenium task above. Please note that I have not shared this directly with Nant or NantContrib. Key features of this task include:

Specify which browser (browserExe) you care about (e.x.: IE versus FireFox, ...)
Specify the suitesuite (testSuiteUrl) to execute your tests
Specify how long to wait (maximumWaitTimeInSeconds) to avoid infinite waiting
Specify if you care about errors or not (failonerror)

There is a lot more to consider to get a reliable Selenium CI environment working (i.e. configuring a database, visiting the correct website address, etc), but the above will help get your foot in the door to bring automated QA testing a giant step above merely running developer tests.

Additional Resources

Cruise Control for Selenium - http://openqa.org/selenium/testrunner.html
Nant - http://nant.sourceforge.net/
How to write a custom nant task - http://blogs.geekdojo.net/rcase/archive/2005/01/06/5971.aspx
CruiseControl.Net - http://confluence.public.thoughtworks.org/display/CCNET/

Additional Source Code

SeleniumTask.cs

using System;
using System.Diagnostics;
using System.IO;
using System.Threading;
using System.Xml;
using NAnt.Core;
using NAnt.Core.Attributes;

namespace NAnt.BuildSpecific.Tasks
{

  [TaskName("selenium")]
  public class SeleniumTask : LoggedTask
  {

    #region Member Variables
    //````````````````````````
    // MEMBER VARIABLES
    //````````````````````````

    private string _browserExe;
    private int _maximumWaitTimeInSeconds;
    private string _testSuiteUrl;
    private string _resultsFile;

    #endregion

    #region Constructor
    //````````````````````````
    // CONSTRUCTOR
    //````````````````````````

    public SeleniumTask()
    {
      init();
    }

    private void init()
    {
      _browserExe = @"c:\Program Files\Mozilla Firefox\firefox.exe";
      _resultsFile = "TestResults.xml";
      _testSuiteUrl = "http://localhost/selenium/TestRunner.html?auto=true&resultsUrl=tests/HandleResults.php";
      
      _maximumWaitTimeInSeconds = 1000;
    }

    #endregion

    #region Properties
    //````````````````````````
    // PROPERTIES
    //````````````````````````

    /// <summary>
    /// Which browse should I run this in?
    ///  + IE
    ///  + FireFox
    ///  
    /// The provided value should be the path to the exe
    /// </summary>
    [TaskAttribute("browserExe", Required=false)]
    public string BrowserExe
    {
      get
      {
        return _browserExe;
      }
      set
      {
        _browserExe = value;
      }
    }

    /// <summary>
    /// How long should we wait for the Selenium tests to complete?
    /// </summary>
    [TaskAttribute("maximumWaitTimeInSeconds", Required=false)]
    public int MaximumWaitTimeInSeconds
    {
      get
      {
        return _maximumWaitTimeInSeconds;
      }
      set
      {
        _maximumWaitTimeInSeconds = value;
      }
    }

    /// <summary>
    /// Where will the results be stored?
    /// </summary>
    [TaskAttribute("resultsFile", Required=false)]
    public string ResultsFile
    {
      get
      {
        return _resultsFile;
      }
      set
      {
        _resultsFile = value;
      }
    }


    /// <summary>
    /// Where should we go to run the Selenium tests?
    /// </summary>
    [TaskAttribute("testSuiteUrl", Required=false)]
    public string TestSuiteUrl
    {
      get
      {
        return _testSuiteUrl;
      }
      set
      {
        _testSuiteUrl = value;
      }
    }

    

    #endregion

    #region Interface
    //````````````````````````
    // INTERFACE
    //````````````````````````

    /// <summary>
    /// 1) Open the browser
    /// 2) Close the browser
    /// </summary>
    protected override void ExecuteTask() 
    {
      PerformExecuteTask();
    }

    public void PerformExecuteTask()
    {

      IntroduceTask();

      bool didComplete = false;
      File.Delete(_resultsFile);

      if (_browserExe != null)
      {
        Process browser = System.Diagnostics.Process.Start(_browserExe,_testSuiteUrl);
        for (int i=0; i<_maximumWaitTimeInSeconds; i++)
        {
          Thread.Sleep(1000);
          if (File.Exists(_resultsFile))
          {
            didComplete = true;
            break;
          }
        }
      }

      if (!didComplete)
      {
        FailAsRequired(string.Format("Selenium tests did not complete within the maximum {0} seconds _
             [Change maximumWaitTimeInSeconds if you would like more time]",_maximumWaitTimeInSeconds));  
      }

      DisplayResults();
      AssertTestsAllPassed();
    }

    public void DisplayResults()
    {
      StreamReader reader = null;
      try
      {
        reader = File.OpenText(_resultsFile);
        string results = reader.ReadLine();

        XmlDocument document = new XmlDocument();
        document.LoadXml(results);

        XmlNode seleniumNode = document.SelectSingleNode(@"/selenium");

        string passes = seleniumNode.Attributes["successes"].Value;
        string failures = seleniumNode.Attributes["failures"].Value;
        string errors = seleniumNode.Attributes["errors"].Value;

        LogInfo(string.Format("{0} success(es), {1} failure(s), {2} error(s)",passes,failures,errors));
      }
      catch (NullReferenceException)
      {
        LogUnknownResults();
      }
      catch (FileNotFoundException)
      {
        LogUnknownResults();
      }
      catch (XmlException)
      {
        LogUnknownResults();
      }
      finally
      {
        if (reader!=null)
        {
          reader.Close();
        }
      }
    }

    public void IntroduceTask()
    {
      LogInfo("Selenium Testing");
      LogInfo(string.Format("TestSuite Url: {0}",_testSuiteUrl));
      LogInfo(string.Format("Results: {0}",_resultsFile));
      LogInfo(string.Format("Browser: {0}",_browserExe));
      LogInfo(string.Format("Will wait upto: {0} second(s)",_maximumWaitTimeInSeconds));
    }

    public void AssertTestsAllPassed()
    {
      StreamReader reader = null;
      try
      {
        reader = File.OpenText(_resultsFile);
        string results = reader.ReadLine();
        
        if (results == null || results.IndexOf("passed") == -1)
        {
          FailAsRequired("Selenium tests failed");
        }
      }
      catch (FileNotFoundException)
      {
        FailAsRequired(string.Format("Selenium results not available in {0} _ 
            [Change resultsFile to fix this issue]",_resultsFile));
      }
      finally
      {
        if (reader!=null)
        {
          reader.Close();
        }
      }
    
    }


    #endregion

    #region Private Methods
    //````````````````````
    // PRIVATE METHODS
    //````````````````````

    private void LogUnknownResults()
    {
      LogInfo("?? success(es), ?? failure(s), ?? error(s)");
    }

    private void FailAsRequired(string text)
    {
      if (FailOnError)
      {
        throw new BuildException(text);
      }
    }

    #endregion
  }
}

LoggedTask.cs

using System;

using NAnt.Core;
using NAnt.Core.Tasks;
using NAnt.Core.Attributes;

namespace NAnt.BuildSpecific.Tasks
{
  /// <summary>
  /// Summary description for LoggedTask.
  /// </summary>
  public class LoggedTask : Task
  {

    #region Member Variables
    //````````````````````````
    // MEMBER VARIABLES
    //````````````````````````

    private NantHelper _helper;

    #endregion

    #region Constructor
    //````````````````````````
    // CONSTRUCTOR
    //````````````````````````

    public LoggedTask()
    {
      init();
    }

    private void init()
    {
      _helper = new NantHelper(this);
    }
  

    #endregion

    #region Interface
    //````````````````````````
    // INTERFACE
    //````````````````````````

    public void CreateProperty(string propertyName, string propertyValue)
    {
      _helper.CreateProperty(propertyName,propertyValue);
    }

    public string GetCreatedProperty(string propertyName)
    {
      return _helper.GetCreatedProperty(propertyName);
    }

    public string GetLogInfoAt(int index)
    {
      return _helper.GetLogAt(index);
    }
    
    public string DerivedPathOf(string path)
    {
      return _helper.DerivedPathOf(path);
    }

    public void LogInfo(string message)
    {
      _helper.Log(message);
    }

    protected override void ExecuteTask()
    {

    }


    #endregion

  }
}

SeleniumTaskTest.cs

using System.IO;
using NAnt.BuildSpecific.Tasks;
using NAnt.Core;
using NUnit.Framework;

namespace Testsuite.Unit.NAnt.BuildSpecific.Tasks
{

  [TestFixture]
  public class SeleniumTaskTest
  {

    private SeleniumTask _task;

    [SetUp]
    public void SetUp()
    {
      File.CreateText("Deleteme.bat").Close();
      _task = new SeleniumTask();
    }

    [TearDown]
    public void TearDown()
    {
      File.Delete("TestResults.xml");
      File.Delete("Deleteme.bat");
    }




    [Test]
    public void BrowserExeSetGet()
    {
      Assert.AreEqual(@"c:\Program Files\Mozilla Firefox\firefox.exe",_task.BrowserExe);
      _task.BrowserExe = "x";
      Assert.AreEqual("x",_task.BrowserExe);
    }

    [Test]
    public void MaximumWaitTimeInSeconds()
    {
      Assert.AreEqual(1000,_task.MaximumWaitTimeInSeconds);
      _task.MaximumWaitTimeInSeconds = 100;
      Assert.AreEqual(100,_task.MaximumWaitTimeInSeconds);
    }

    [Test]
    public void TestSuiteUrl()
    {
      Assert.AreEqual("http://localhost/selenium/TestRunner.html?auto=true&resultsUrl=tests/HandleResults.php",_task.TestSuiteUrl);
      _task.TestSuiteUrl = "x";
      Assert.AreEqual("x",_task.TestSuiteUrl);
    }

    [Test]
    public void ResultsFile()
    {
      Assert.AreEqual("TestResults.xml",_task.ResultsFile);
      _task.ResultsFile = "x";
      Assert.AreEqual("x",_task.ResultsFile);
    }

    [Test]
    [ExpectedException(typeof(BuildException),"Selenium tests did not complete within the maximum 0 seconds _ 
       [Change maximumWaitTimeInSeconds if you would like more time]")]
    public void FailIfResultsDoNotAppear()
    {
      _task.TestSuiteUrl = "x";
      _task.MaximumWaitTimeInSeconds = 0;
      _task.BrowserExe = null;
      _task.PerformExecuteTask();
    }
  
    [Test]
    [ExpectedException(typeof(BuildException),"Selenium tests did not complete within the maximum 0 seconds _ 
        [Change maximumWaitTimeInSeconds if you would like more time]")]
    public void EnsureToDeleteResultsFile()
    {
      File.CreateText("TestResults.xml").Close();
      _task.TestSuiteUrl = "Deleteme.bat";
      _task.MaximumWaitTimeInSeconds = 0;
      _task.BrowserExe = null;
      _task.PerformExecuteTask();
      Assert.AreEqual(false,File.Exists("TestResults.xml"));
    }

    [Test]
    [ExpectedException(typeof(BuildException),"Selenium results not available in x _ 
       [Change resultsFile to fix this issue]")]
    public void AssertTestsAllPassed_NoFile()
    {
      _task.ResultsFile = "x";
      _task.AssertTestsAllPassed();
    }

    [Test]
    [ExpectedException(typeof(BuildException),"Selenium tests failed")]
    public void AssertTestsAllPassed_NoPassWihtinFile()
    {
      File.CreateText("TestResults.xml").Close();
      _task.AssertTestsAllPassed();
    }

    [Test]
    [ExpectedException(typeof(BuildException),"Selenium tests failed")]
    public void AssertTestsAllPassed_InvalidText()
    {
      createResultsFileWithText("invalid");
      _task.AssertTestsAllPassed();
    }

    [Test]
    [ExpectedException(typeof(BuildException),"Selenium tests failed")]
    public void AssertTestsAllPassed_Failed()
    {
      createResultsFileWithText("<selenium result=\"failed\" totalTime=\"2\" successes=\"3\" failures=\"1\" errors=\"0\" />");
      _task.AssertTestsAllPassed();
    }

    [Test]
    public void AssertTestsAllPassed_ButFailOnErrorIsFalse_Failed()
    {
      _task.FailOnError = false;
      createResultsFileWithText("<selenium result=\"failed\" totalTime=\"2\" successes=\"3\" failures=\"1\" errors=\"0\" />");
      _task.AssertTestsAllPassed();
    }

    [Test]
    public void AssertTestsAllPassed_Okay()
    {
      createResultsFileWithText("<selenium result=\"passed\" totalTime=\"2\" successes=\"3\" failures=\"1\" errors=\"0\" />");
      _task.AssertTestsAllPassed();
    }

    [Test]
    public void IntroduceTask_CheckLog()
    {
      createResultsFileWithText("<selenium result=\"passed\" totalTime=\"2\" successes=\"3\" failures=\"1\" errors=\"0\" />");

      _task.TestSuiteUrl = "x";
      _task.ResultsFile = "y";
      _task.MaximumWaitTimeInSeconds = 3;
      _task.BrowserExe = "w";
      
      _task.IntroduceTask();

      Assert.AreEqual("Selenium Testing",_task.GetLogInfoAt(0));
      Assert.AreEqual("TestSuite Url: x",_task.GetLogInfoAt(1));
      Assert.AreEqual("Results: y",_task.GetLogInfoAt(2));
      Assert.AreEqual("Browser: w",_task.GetLogInfoAt(3));
      Assert.AreEqual("Will wait upto: 3 second(s)",_task.GetLogInfoAt(4));

    }

    [Test]
    public void DisplayResults_NoFile_Log()
    {
      _task.DisplayResults();
      Assert.AreEqual("?? success(es), ?? failure(s), ?? error(s)",_task.GetLogInfoAt(0));
    }


    [Test]
    public void DisplayResults_InvalidFile_Log()
    {
      createResultsFileWithText("x");
      _task.DisplayResults();
      Assert.AreEqual("?? success(es), ?? failure(s), ?? error(s)",_task.GetLogInfoAt(0));
    }


    [Test]
    public void DisplayResults_Log()
    {
      createResultsFileWithText("<selenium result=\"passed\" totalTime=\"2\" successes=\"3\" failures=\"1\" errors=\"0\" />");
      _task.DisplayResults();
      Assert.AreEqual("3 success(es), 1 failure(s), 0 error(s)",_task.GetLogInfoAt(0));
    }

    private void createResultsFileWithText(string text)
    {
      StreamWriter writer = null;
      try
      {
        writer = File.CreateText("TestResults.xml");
        writer.Write(text);
      }
      finally
      {
        if (writer != null)
        {
          writer.Close();
        }
      }
    }


  }
}