Distributed Hibernate Search with Apache Tomcat 6, ActiveMQ and Spring

Today I would like to share my experience with configuring Hibernate Search in master/slave(s) deployment using Apache ActiveMQ, Spring and running all this stuff inside Apache Tomcat 6 container.

How it works:
- Hibernate Search supports distributed configuration using JMS back-end and master / slave(s) index
- master server exposes index over network share (NFS,...)
- slave(s) on regular base replicate this master copy to own local copies

Used version:
- Apache Tomcat 6.0.20
- Hibernate Search 3.1.1 GA
- Apache ActiveMQ 5.3.0
- Spring 2.5.6
- XBean-Spring 3.6

Master Index Configuration
Master configuration is a little bit complicated. Here are configuration-specific properties:

${local.index.dir} - directory to store master index
${master.index.dir} - directory to copy master index to, it's shared network location for replication with slave(s)

First of all, for simplification, let's run ActiveMQ broker on the same host. For this purpose we could use simple embedded broker configuration placed into WEB-INF/activemq.xml:

<beans
 xmlns="http://www.springframework.org/schema/beans"
 xmlns:amq="http://activemq.apache.org/schema/core"
 xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
 xsi:schemaLocation="http://www.springframework.org/schema/beans
 http://www.springframework.org/schema/beans/spring-beans-2.0.xsd
 http://activemq.apache.org/schema/core
 http://activemq.apache.org/schema/core/activemq-core.xsd">

 <amq:broker brokerName="HibernateSearchBroker">
     <amq:managementContext>
         <amq:managementContext createConnector="false"/>
     </amq:managementContext>

     <amq:transportConnectors>
         
     </amq:transportConnectors>
  </amq:broker>

  <amq:queue name="queue/hibernatesearch" physicalName="hibernateSearchQueue" />
</beans>

Then we need to configure JNDI resources (JMS Connection Factory and Queue) through web application META-INF/context.xml file (Tomcat-specific):

...
<!-- ActiveMQ ConnectionFactory -->
<Resource
name="jms/ConnectionFactory"
auth="Container"
type="org.apache.activemq.ActiveMQConnectionFactory"
description="JMS Connection Factory"
factory="org.apache.activemq.jndi.JNDIReferenceFactory"
brokerURL="tcp://0.0.0.0:61616?trace=true"
brokerName="HibernateSearchBroker" />

<!-- ActiveMQ HibernateSearch queue -->
<Resource
name="queue/hibernatesearch"
auth="Container" type="org.apache.activemq.command.ActiveMQQueue"
description="Hibernate search queue"
factory="org.apache.activemq.jndi.JNDIReferenceFactory"
physicalName="hibernateSearchQueue" />
...

Next step is configuration of the Hibernate Search itself through Hibernate configuration file (hibernate.cfg.xml):

<property name="hibernate.search.default.directory_provider">org.hibernate.search.store.FSMasterDirectoryProvider</property>
<property name="hibernate.search.default.indexBase">${local.index.dir}</property>
<property name="hibernate.search.default.sourceBase">${master.index.dir}</property>
<property name="hibernate.search.default.refresh">60</property>

One important difference between master and slave codebase: master implementation must include subclass of AbstractJMSHibernateSearchController as message listener. For example,

import javax.jms.MessageListener;

import org.hibernate.Session;
import org.hibernate.search.backend.impl.jms.AbstractJMSHibernateSearchController;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;

@Service
public class JMSHibernateSearchController
 extends AbstractJMSHibernateSearchController
     implements MessageListener {

 @Override
 protected void cleanSessionIfNeeded(Session session) {
    // clean session here ...     
 }

 @Override
 protected Session getSession() {
    // return new session here ...     
 }
}

Finally, let's wrap it up inside Spring configuration file applicationContext.xml:

<bean id="broker" class="org.apache.activemq.xbean.BrokerFactoryBean">
 <property name="config" value="WEB-INF/activemq.xml" />
 <property name="start" value="true" />
</bean>  

<bean name="jmsConnectionFactory" class="org.springframework.jndi.JndiObjectFactoryBean">
 <property name="jndiName" value="java:comp/env/jms/ConnectionFactory" />
</bean>

<bean name="jmsHibernateSearchQueue" class="org.springframework.jndi.JndiObjectFactoryBean">
 <property name="jndiName" value="java:comp/env/queue/hibernatesearch" />

<bean id="hibernateSearchController" class="<your implementation of AbstractJMSHibernateSearchController>" />

<bean id="jmsContainer" class="org.springframework.jms.listener.DefaultMessageListenerContainer" depends-on="broker">
 <property name="connectionFactory" ref="jmsConnectionFactory"/>
 <property name="destination" ref="jmsHibernateSearchQueue"/>
 <property name="messageListener" ref="hibernateSearchController" />
</bean>

With those configurations in place Hibernate Search master is ready to run.


Slave Index Configuration
Slave(s) configuration is much simple. Here are configuration-specific properties:

${server} - server which runs ActiveMQ broker
${local.index.dir} - directory to store local index (master copy)
${master.index.share} - mounted network share with master index

First of all, we need to configure JNDI resources (JMS Connection Factory and Queue) through web application META-INF/context.xml file (Tomcat-specific):

...
<!-- ActiveMQ ConnectionFactory -->
<Resource
name="jms/ConnectionFactory"
auth="Container"
type="org.apache.activemq.ActiveMQConnectionFactory"
description="JMS Connection Factory"
factory="org.apache.activemq.jndi.JNDIReferenceFactory"
brokerURL="tcp://${server}:61616?trace=true"
brokerName="HibernateSearchBroker" />

<!-- ActiveMQ HibernateSearch queue -->
<Resource
name="queue/hibernatesearch"
auth="Container" type="org.apache.activemq.command.ActiveMQQueue"
description="Hibernate search queue"
factory="org.apache.activemq.jndi.JNDIReferenceFactory"
physicalName="hibernateSearchQueue" />
...

Then we have to configure Hibernate Search itself through Hibernate configuration file (hibernate.cfg.xml):

<property name="hibernate.search.default.directory_provider">org.hibernate.search.store.FSSlaveDirectoryProvider</property>
<property name="hibernate.search.default.indexBase">${local.index.dir}</property>
<property name="hibernate.search.default.sourceBase">${master.index.share}</property>
<property name="hibernate.search.default.refresh">60</property>
<property name="hibernate.search.worker.backend">jms</property>
<property name="hibernate.search.worker.jms.connection_factory">java:comp/env/jms/ConnectionFactory</property>
<property name="hibernate.search.worker.jms.queue">java:comp/env/queue/hibernatesearch</property>
<property name="hibernate.search.worker.jndi.java.naming.factory.initial">org.apache.activemq.jndi.ActiveMQInitialContextFactory</property>

And ... that's it!

Few additional words about testing all this stuff with JUnit. Only problem is JNDI which could be mocked up with Spring JNDI templates. For example:

<bean name="jmsConnectionFactory" class="org.springframework.jndi.JndiObjectFactoryBean">
    <property name="jndiName" value="java:comp/env/jms/ConnectionFactory" />
    <property name="jndiTemplate">
        <bean class="org.springframework.mock.jndi.ExpectedLookupTemplate">
            <constructor-arg index="0" value="java:comp/env/jms/ConnectionFactory" />
            <constructor-arg index="1">
                <bean class="org.apache.activemq.ActiveMQConnectionFactory">
                    <property name="brokerURL">
                        <value>tcp://0.0.0.0:61616</value>
                    </property>
                </bean>
            </constructor-arg>
        </bean>
    </property>
</bean>

<bean name="jmsHibernateSearchQueue" class="org.springframework.jndi.JndiObjectFactoryBean">
    <property name="jndiName" value="java:comp/env/queue/hibernatesearch" />
    <property name="jndiTemplate">
       <bean class="org.springframework.mock.jndi.ExpectedLookupTemplate">
           <constructor-arg index="0" value="java:comp/env/queue/hibernatesearch" />
           <constructor-arg index="1">
               <bean id="jmsHibernateSearchQueue"   class="org.apache.activemq.command.ActiveMQQueue">
                   <constructor-arg value="queue/hibernateSearchQueue"/>
               </bean>               
           </constructor-arg>
       </bean>
    </property>
</bean>
...