The adventures of a software architect

Having fun with Scala, part 1

Fri, 30 Sep 2011 11:07:49 +0200

Developing enterprise applications there is a recurring problem, how to simply manage orthogonal concerns like auditing, logging, authentication, etc. A typical approach is just using an AOP technology, sure, AOP it works but sometime it’s just an overkilling. In my experience, often, we just need a simple mechanism to intercept calls in the service layer, i.e. in the implementation of a REST/WS interface.
So, as an exercise, I tried to implement a simple interception framework using Scala, and since the code worths thousands words here you are the result of my effort:

package it.davidgreco.scalaexperiments

import collection.mutable.HashMap
import java.lang.reflect.Method

private[scalaexperiments] trait ReadWriteLock {
private val lock = new java.util.concurrent.locks.ReentrantReadWriteLock

def writeLock[S](f: => S): S = {
lock.writeLock.lock()
try {
f
}
finally {
lock.writeLock.unlock()
}
}

def readLock[S](f: => S): S = {
lock.readLock.lock()
try {
f
}
finally {
lock.readLock.unlock()
}
}
}

object Interceptable extends ReadWriteLock {

type PreInterceptor = Interceptable => Unit
type PostInterceptor = PreInterceptor
type AroundInterceptor = Interceptable => (=> Any) => Any

var preInterceptors = new Array[Option[PreInterceptor]](5)
var preInterceptorsNamePosition = new HashMap[String, Int]

var postInterceptors = new Array[Option[PostInterceptor]](5)
var postInterceptorsNamePosition = new HashMap[String, Int]

var aroundInterceptors = new Array[AroundInterceptor](5)
var aroundInterceptorsNamePosition = new HashMap[String, Int]

def registerPreInterceptor(pos: Int, name: String, interceptor: PreInterceptor): Unit = {
writeLock {
preInterceptorsNamePosition += name -> pos
preInterceptors.update(pos, Some(interceptor))
}
}

def unregisterPreInterceptor(pos: Int) {
writeLock {
preInterceptors.update(pos, None)
val name = (for (p <- preInterceptorsNamePosition; if (p._2 == pos)) yield p._1).toList.head
preInterceptorsNamePosition.remove(name)
}
}

def unregisterPreInterceptor(name: String) {
writeLock {
val p = preInterceptorsNamePosition.get(name)
for (pos <- p) {
preInterceptors.update(pos, None)
preInterceptorsNamePosition.remove(name)
}
}
}

def registerPostInterceptor(pos: Int, name: String, interceptor: PostInterceptor): Unit = {
writeLock {
postInterceptorsNamePosition += name -> pos
postInterceptors.update(pos, Some(interceptor))
}
}

def unregisterPostInterceptor(pos: Int) {
writeLock {
postInterceptors.update(pos, None)
val name = (for (p <- postInterceptorsNamePosition; if (p._2 == pos)) yield p._1).toList.head
postInterceptorsNamePosition.remove(name)
}
}

def unregisterPostInterceptor(name: String) {
writeLock {
val p = postInterceptorsNamePosition.get(name)
for (pos <- p) {
postInterceptors.update(pos, None)
postInterceptorsNamePosition.remove(name)
}
}
}

def registerAroundInterceptor(pos: Int, name: String, interceptor: AroundInterceptor): Unit = {
writeLock {
aroundInterceptorsNamePosition += name -> pos
aroundInterceptors.update(pos, interceptor)
}
}

def unregisterAroundInterceptor(pos: Int) {
writeLock {
aroundInterceptors.update(pos, identityAroundInterceptor)
val name = (for (p <- aroundInterceptorsNamePosition; if (p._2 == pos)) yield p._1).toList.head
aroundInterceptorsNamePosition.remove(name)
}
}

def unregisterAroundInterceptor(name: String) {
writeLock {
val p = aroundInterceptorsNamePosition.get(name)
for (pos <- p) {
aroundInterceptors.update(pos, identityAroundInterceptor)
aroundInterceptorsNamePosition.remove(name)
}
}
}

private[Interceptable] def identityAroundInterceptor[A](interceptable: Interceptable)(x: => A): A = {
x
}

for (i <- 0 until preInterceptors.size) {
preInterceptors.update(i, None)
}

for (i <- 0 until postInterceptors.size) {
postInterceptors.update(i, None)
}

for (i <- 0 until aroundInterceptors.size) {
aroundInterceptors.update(i, identityAroundInterceptor)
}
}

trait Interceptable {

self =>

import Interceptable._

def intercept[A](block: => A): A = {

/*
val exc = new Exception
val st = exc.getStackTrace
val ste = st.apply(2)

val cls = Class.forName(ste.getClassName())
var back: Method = null

cls.getDeclaredMethods().foreach {
m: Method =>
if (back == null && m.getName() == ste.getMethodName()) {
back = m
}
}
*/

readLock {
for (f <- Interceptable.preInterceptors; if (f.isDefined)) {
f.get(this)
}
}

val a = readLock {
aroundInterceptors.apply(4)(self)(aroundInterceptors.apply(3)(self)(aroundInterceptors.apply(2)(self)(aroundInterceptors.apply(1)(self)(aroundInterceptors.apply(0)(self)(block)))))
}

readLock {
for (f <- Interceptable.postInterceptors; if (f.isDefined)) {
f.get(this)
}
}

a.asInstanceOf[A]
}

}

class MyClass extends Interceptable {

def method(i: Int): Unit = intercept {
println("Method " + i)
}

def methodWithInterception = intercept {}

def methodWithoutInterception = {}

}

object Main {

import Interceptable._

def main(args: Array[String]) {

registerPreInterceptor(1, "MyPreInterceptor", (x: Interceptable) => {
println("PRE_INTERCEPTED")
})

registerPostInterceptor(1, "MyPostInterceptor", (x: Interceptable) => {
println("POST_INTERCEPTED")
})

def dummyAroundInterceptor1[A](interceptable: Any)(x: => A): A = {
println("BEFORE_1");
val a = x;
println("AFTER_1");
a
}

def dummyAroundInterceptor2[A](interceptable: Any)(x: => A): A = {
println("BEFORE_2");
val a = x;
println("AFTER_2");
a
}

registerAroundInterceptor(0, "DUMMY1", dummyAroundInterceptor1)
registerAroundInterceptor(1, "DUMMY2", dummyAroundInterceptor2)

val a = new MyClass

a.method(1)

unregisterAroundInterceptor(0)
unregisterAroundInterceptor(1)

unregisterPreInterceptor(1)
unregisterPostInterceptor(1)

a.method(2)

val start = System.currentTimeMillis()
for (i <- 1 to 1000000) {
a.methodWithoutInterception
}
val stop1 = System.currentTimeMillis()
println(stop1 - start)

for (i <- 1 to 1000000) {
a.methodWithInterception
}
val stop2 = System.currentTimeMillis()
println(stop2 - stop1)

println(stop2 - 2 * stop1 + start)

}

}

I’m pretty sure that there are better ways for doing the same thing, I’m still on the learning curve and I would really like to have your comments on how this could be improved.
What I like is the fact that I found a way to implement also the around interceptor mechanism which is pretty useful if, for example, you want to measure the execution time for a method call.
Anyway, Scala is such a great language!

Scala and JPA: a couple of hints

Wed, 16 Feb 2011 17:43:09 +0100

At work we are developing an application using Scala, during the design phase we had to decide which ORM was better for our purposes. Our first choice was Squeryl, it;s a pure Scala based ORM and it’s great, however we thought that it was still on a early stage of development and we didn’t want to add additional risk to our development (we are still a young Scala shop here). Then, we opted to use JPA.
When first I tried to combine Scala with JPA it was pretty weird, Scala 2.7.x used to have a bad support for Java annotations. Now, it works perfectly and what I really like is that it’s possible to use JPA together with immutable case classes, I don’t want to repeat how good is to use immutable objects especially in the functional world. Even with JPA entities we have the full power of pattern matching, really great!
After some investigation I and my colleague Martin we found the solution.
Below I put just a quick example of an immutable case class with the needed JPA annotations:

@Entity
@Table(name="devices")
case class Device(

@(Id @field)
@(GeneratedValue @field)
@(Column @field)(columnDefinition="int(11)")
id: Long,

@(Column @field)(unique=true, columnDefinition="varchar(255)")
@(Index @field)(name="index_devices_on_machine_id")
deviceMachineId: String,

@(Type @field)
@(Column @field)(columnDefinition="int(10)")
fcd: Long {

def this() = this (0L, "", 0L)
}

the trick is to use the Scala nested annotations to force the compiler to generate setters and getters even if the case class is immutable, @field annotation does that.
And you have also to add a default constructor. Pretty simple no?

I started climbing the Akka mountain

Thu, 6 Jan 2011 18:01:11 +0100

One of the reasons why I like the Scala language is the fact that in the Scala community you can find people like Jonas Bonér and Viktor Klang. Those people created a great framework: Akka
The Akka framework provides an extremely fast and scalable actor implementation. It provides also other interesting goodies, but I don’t want to bother with too much details the Akka site is plenty of documentation and it’s not so difficult to find examples around.
I had the chance to contribute to the Akka framework and for me it was really nice to be able to add a couple of small pieces to it.
One of the things I really appreciate of Akka is the easiness in setting up a Scala project based on SBT (Simple Build Tool) and Idea as the IDE thanks to this wonderful sbt plugin sbt-idea.
I think that the combination of SBT and Idea is the best environment for developing Scala based applications.
In this post I’ll give you some hints on how to quickly set up an Akka based project using SBT as a building tool and Idea as the preferred tool.
So, let’s start with the various steps. On a shell start to type the following commands (Unix/Linux):
1) mkdir akka-template
2) cd akka-template
3) sbt
Running the command above, sbt will ask you a couple of questions, you can just reply: y, akka-template, myorganization for example, the meaning is pretty obvious.
1) Then, under the sbt console, just type exit

You should still be under the akka-template directory, so just create two additional directories (under the akka-template directory):
1) mkdir project/build
2) mkdir project/plugins

Under the directory akka-template/project/build put a file names akka-template.scala for example with the following content:

import sbt._

class AkkaTemplate(info: ProjectInfo) extends DefaultWebProject(info) with AkkaProject {

val akka_repo = "Akka Maven Repository" at "http://akka.io/repository"

val akkaHttp = akkaModule("http")

val jetty7Server = "org.eclipse.jetty" % "jetty-server" % "7.0.1.v20091125" % "test"
val jetty7WebApp = "org.eclipse.jetty" % "jetty-webapp" % "7.0.1.v20091125" % "test"
}

This file represents the actual project build configuration for sbt.
Then, under the directory akka-template/project/plugins put a file named Plugins.scala with the following content:

import sbt._

class Plugins(info: ProjectInfo) extends PluginDefinition(info) {
val akka_repo = "Akka Maven Repository" at "http://akka.io/repository"
val akkaPlugin = "se.scalablesolutions.akka" % "akka-sbt-plugin" % "1.0-RC2"
}

This file allows adding additional SBT plugins useful for doing specific building actions. In this specific example you are including the Akka specific plugin.
This specific plugin provides an AkkaProject trait which allows to add Akka modules with their dependencies automatically into your project. In the file akka-template.scala above, you can notice the line:
val akkaHttp = akkaModule("http")
that means that the web project is using the Akka REST module. In this way you have everything for implementing a REST based application with all the JAX-RS goodies, backed by the powerful Akka actors.
The last step is to run again sbt under the akka-template directory. Under the sbt shell type:
> idea

That sbt action will create all the needed artifacts for the Idea IDE. Now you can open the project under Idea, that’s it.
I really hope it will be useful for you to start climbing the great Akka mountain ;)

My stairway to the cloud

Fri, 2 Jul 2010 13:49:18 +0200

In the last couple of weeks, probably influenced by the hype around that word, I decided to learn a little bit more about cloud computing.
I was already a subscriber of the Amazon EC2 service but I never got the chance to play with it, so I decided to give it a try.
My experience with EC2 has been very positive and I had also the chance to play a lot with the Eucalyptus product which is a sort of “clone” of EC2.
Frankly speaking the word “Cloud” is really over exposed, too much marketing, too much business “noise” around, it remembers me when SOA was like the cure for every IT disease, but like SOA, cloud computing is just a paradigm shift, it’s an evolution of something we have already, like SOA is an architectural pattern evolved from the concept of component based architecture.
I think that cloud computing is just an evolution of the concept of virtualization which is not new, the concept of virtual machine is around since the advent of the IBM VM370 architecture in the early 70’.
Modern cloud infrastructures provide very nice and powerful automation infrastructure around some kind of hypervisor/virtualization kernel.
This is the real value of the cloud, easy to use automation mechanisms for provisioning of virtual resources: machines, disks, networks.
The paradigm shift, I think, is in the fact that the cloud move towards the programmers what is used to be in the hands of system administrators.
Using a private/public cloud, you, as a programmer, can put into your development life cycle the management of the infrastructure needed to run your application. As it happen often, if you want to really get the advantages of a technology you have to pay a price. In the case of the cloud, if your application has not been designed to be “elastic”, i.e. easy to scale and deploy, you don’t get the real bonus in using a cloud.
An application made of many different kind of subsystems with complex and rigid relationships among them it doesn’t fit very well with a cloud infrastructure. A rigid application with a dynamic infrastructure cannot leave well together no ?
There are also “social” consequences in the cloud adoption. If you start to use a cloud either private or public, you could also take the ownership of the physical deployment which is used to be in the sphere of influence of the sys admins. I observed some diffidence in the operational people when I introduced a private cloud into my organization, they thought that it was a sort of lost of power, they used to be engaged even for creating ad hoc virtual machines for doing stress tests for example. With a private cloud in your laboratory, the developers can create by themselves everything they need for doing integration and scalability tests.
Fortunately, the system and network administrators in my company are smart open to new things, so they realized that the cloud can free them from boring provisioning activities, giving them extra time in looking after the “iron”, the actual hardware, the storage, the network elements, etc.
Off course, the cloud allows a better utilization of the hardware, so with the same hardware you can do more. Combining a private cloud with a public one, you could absorb the load peaks just temporary provisioning on the public cloud, in this way you are note necessarily forced to buy all the hardware needed by the foreseen peak.
So, behind the hype I really think that cloud computing it’s another nice tool for us, as software architects, for designing applications easy to scale and to adapt themselves automatically to the load changes.
As a side note, I wanted to learn a little bit more the language Scala, so I decided to define a DSL (Domain Specific Language) for quickly provisioning software infrastructure on top of an EC2 based cloud either Amazon or Eucalyptus, you can find the project here. It’s still a working in progress and nothing more than an experiment, but at least I could learn a bit more of Scala and how to deal with EC2 and Eucalyptus.

HPC = High Performance Camel ?

Mon, 19 Apr 2010 11:27:12 +0200

I did another experiment with my favorite software beast. Even if I’m still convinced that using C/C++ for building low latency/high throughput applications is more effective than using Java I’m also convinced that this should be balanced with the cost of development. So, provided that Java allows a more effective way for developing complex applications, I started to wonder if Camel can be also used as a bus in scenarios where performances and scalability really count.
As in the previous post, I made some experiments integrating Camel with a very fast messaging library and the preliminary results are not too bad I reached a throughput of more than 50K messages per second (1KB payload). The next step was to see if Camel can be used for storing messages into an highly scalable NoSQL store like HBase. My first idea was to write a Camel component for reading/writing data into HBase as I did with the camel-hdfs component, since I’m lazy I looked around to see if something similar was already made by someone else. This is what I found:
1. http://ghelmling.github.com/meetup.beeno
2. http://www.datanucleus.org/plugins/store.hbase.html
I decided to investigate a little bit more with the second option. In fact the Datanucleus guys provide us with a JPA/JDO tool with a pluggable storage mechanism where one of the different storages can be HBase.
Using Datanucleus you can just annotate a class using either the JPA or JDO standard annotations and voila that class can be automatically persisted into HBase. Additionally, Camel provides a camel-jpa component which allows to to store and retrieve Java objects from persistent storage using JPA. Combining camel-jpa and the Datanucleus Hbase plugin I have a camel based mechanism for streaming messages into an HBase store and this is what I needed.
So, I put together a simple example that shows how to stream events to a Camel endpoint, using zeromq and to store them into an HBase store.
It seems to me that Camel is enough lightweight and fast to be used as a bus for glowing together technologies for the realization of high performant/available architectures. In any case, I think that it could be useful to investigate and to see if it’s possible to increase the overall throughput of Camel this could make it a very nice tool even in contexts where the throughput is the most fundamental requirement.
You can find the complete maven based project here.

A quick Camel ride

Sat, 27 Mar 2010 14:00:25 +0100

I wanted to do a quick experiment with Camel. My idea was to find a way for quickly integrating Camel with any native transport library.
As an example I took the zeromq library and I wrote a Camel component which supports that library for sending and receiving byte buffers.
For developing this component I needed to find a way for making portable the integration of a pure Java based framework like Camel with C/C++ based code. Fortunately there are nice tools that allow developing portable JNI C/C++ code able to be compiled seamlessly under different platforms. The result is a maven based project that could be integrated into the Camel building process even if it needs to build native libraries.
It works under Mac OSX and Linux and adding the support for other OSes shouldn’t be so difficult.
The tools I used are:
1. CMAKE: it’s a powerful building files generator.
2. SWIG: it generates all the JNI plumping for integrating existing C/C++ code with Java.
3. BOOST: it’s one of the most important C++ frameworks. It provides among many different things OS independent APIs for concurrent programming.
You can find the project under GitHub.
See you at the next Camel ride.

My Camel likes the Elephant

Sun, 20 Dec 2009 13:00:36 +0100

I’m still working on the Camel HDFS component and I think it reached a good level of maturity. I added also the consumer part, so it’s possible now to read from an HDFS file. I submitted the patch to the Camel community and it’s pending for the approval and to be integrated into the Camel trunk.
In the meanwhile if you wanna play with the latest version of this component, you can check it out from my subversion repository:

svn co http://home.davidgreco.it/svn/camel-hdfs/trunk camel-hdfs

Below the documentation of the component.

HDFS Component:

URI Format

hdfs://hostname[:port][/path][?options]

the path is treated in the following way:
1) as a consumer if it’s a file it just read the file, otherwise if it represents a directory it scans all the file under the path satisfying the configured pattern. All the files under that directory must be of the same type.
2) as a producer, if at least one split strategy is defined, the path is considered a directory and under that directory the producer creates a different file per split named seg0, seg1, seg2, etc.

Options

Name Default Value Description
overwrite true the HDFS file can be overwritten

bufferSize 4096 the buffer size used by HDFS

replication 3 the HDFS replication factor

blockSize 64MB the size of the HDFS blocks

fileType NORMAL_FILE it can be SEQUENCE_FILE,
MAP_FILE, ARRAY_FILE, or
BLOOMMAP_FILE, see Hadoop

fileSystemType HDFS it can be LOCAL for local filesystem

keyType NULL the type for the key in case of
sequence or map files. See below.

valueType TEXT the type for the key in case of
sequence or map files. See below.

splitStrategy A string describing the strategy on
how to split the file based on different
criteria. See below.

openedSuffix opened When a file is opened for reading/
writing the file is renamed with this
suffix to avoid to read it during the
writing phase.

readSuffix read Once the file has been read is
renamed with this suffix to avoid to
read it again.

initialDelay 0 For the consumer, how much to wait
before to start scanning the directory.

delay 0 Then interval between the directory
scans.

pattern * The pattern used for scanning the
directory

chunkSize 4096 When reading a normal file, this is split
into chunks producing a message per
chunk

The keyType and the valueType can be:
NULL it means that the key or the value is absent
BYTE for writing a byte, the java Byte class is mapped into a BYTE
BYTES for writing a sequence of bytes. It maps the java ByteBuffer class
INT for writing java integer
FLOAT for writing java float
LONG for writing java long
DOUBLE for writing java double
TEXT for writing java strings

BYTES is also used with everything else, for example, in Camel a file is sent around as an InputStream, int this case is written in a sequence file or a map file as a sequence of bytes.

Splitting Strategy
In the current version of Hadoop (0.20.1) opening a file in append mode is disabled since it’s not enough reliable. So, for the moment, it’s only possible to create new files. The Camel HDFS endpoint tries to solve this problem in this way:
• If the split strategy option has been defined, the actual file name will be <file name>0 initially
• Every time a splitting condition is met a new file is created with name <original file name>N where N is 1, 2, 3, etc.
The splitStrategy option is defined as a string with the following syntax:
splitStrategy=<ST>:<value>,<ST>:<value>,*

where <ST> can be:
BYTES a new file is created, and the old is closed when the number of written bytes is more than <value>
MESSAGES a new file is created, and the old is closed when the number of written messages is more than <value>
IDLE a new file is created, and the old is closed when no writing happened in the last <value> milliseconds

for example:
hdfs://localhost/tmp/simple-file?splitStrategy=IDLE:1000,BYTES:5
it means: a new file is created either when it has been idle for more than 1 second or if more than 5 bytes have been written

P.S. (8 Jan 2010)
I moved this project to github here. There is also a wiki page under the Apache Camel documentation, here.

When a Camel encounters an Elephant

Tue, 17 Nov 2009 11:59:30 +0100

I don’t know why, but I’m still pretty linked to my HPC background, even if I spent most of my time working for different kind of enterprises and trying to solve problems on different domains, I always thought that enterprises should try to learn from the HPC community how to solve “hard” business problems. With the adjective “hard”, I mean difficult in terms of computational power needed for solving the problem.
I’m observing a sort of convergence towards a sort of cross fertilization in terms of technologies involved between the HPC community and the enterprise business community. For example, grid computing and cloud computing are technologies that used to be already popular in the HPC field, but now are getting more and more interest even in the enterprises.
In the HPC, off course the emphasis is more on the solution of numerical problems where the data set involved can be really huge. There are so many interesting problems in this area: oil exploration, weather forecast, molecular dynamics, etc. On the other end enterprises need to process huge amount of data as well, but in this case they usually need to digest huge amount of data in structured/semi-structured form for extracting something useful: analysis of web servers logs, information retrieval/extraction from a massive number of documents, indexing, etc.
In both cases, there is a trend in trying to scale the applications using many inexpensive nodes than using big computers.
Along this line, I started to give a look to a very interesting technology used for scaling the analysis of very huge quantity of data.
This technology is Hadoop. Hadoop is an open source implementation of the map/reduce algorithm proposed by Google. I would say that Google didn’t invent anything, they just applied the concept of collective computations (reductions), typically used in parallel computing, to the problem of creating reverse indexes for the web search. I used to use sort of map/reduce algorithm using MPI when I was working in a parallel computing laboratory.
I don’t want to go deep into details regarding the Hadoop ecosystem, on the Apache site it’s plenty of information. What I tried to do is to see if one of my favorites open source tools, Camel, could be useful for making the usage of Hadoop easier. Look at the picture below:

A fundamental piece in the Hadoop ecosystem is HDFS (Hadoop File System). From the Apache site: “The Hadoop Distributed File System (HDFS) is a distributed file system designed to run on commodity hardware. It has many similarities with existing distributed file systems. However, the differences from other distributed file systems are significant. HDFS is highly fault-tolerant and is designed to be deployed on low-cost hardware. HDFS provides high throughput access to application data and is suitable for applications that have large data sets.”
So, before analyzing the data the first step is to put the data on the HDFS filesystem. HDFS provides APIs and also command line utilities for loading data into HDFS, however I thought it would be nice to take advantage of the multi-transport/multi-protocol nature of Camel for accomplishing the task of filling the Hadoop file system.
In fact, using Camel you could easily gather data for virtually any kind of system using a plethora of many different mechanisms: files, ftp, http, mail, atom, rss, xmpp, etc. So, having an HDFS Camel component you could just define routes that gather data from any kind of sources to be put directly into the Hadoop file system, this is why I wrote a Camel HDFS component. In attachment I put the zip file containing the component, youi can dig around the tests to see how it works. At the end of this post I’ll put some documentation. For the next steps I’m thinking if it makes sense to integrate Camel even with the map/reduce framework. In the meanwhile I think that using Camel only for writing data into an HDFS filesystem could really help.
Very soon I’ll try to submit this work to the official Camel project.

HDFS Component:
This component provides only a producer, it only allows writing data into the HDFS, reading from HDFS is not implemented yet. This component can be only used with the to clause.

URI Format

hdfs://hostname[:port][/path][?options]

Options

Name Default Value Description
append false the HDFS file is opened in append
mode

overwrite true the HDFS file can be overwritten

bufferSize 4096 the buffer size used by HDFS

replication 3 the HDFS replication factor

blockSize 64MB the size of the HDFS blocks

fileType NORMAL_FILE it can be SEQUENCE_FILE or
MAP_FILE, see Hadoop

fileSystemType HDFS it can be LOCAL for local filesystem

keyType NULL the type for the key in case of
sequence or map files. See below.

valueType TEXT the type for the key in case of
sequence or map files. See below.

splitStrategy A string describing the strategy on
how to split the file based on different
criteria. See below

The keyType and the valueType can be:
NULL it means that the key or the value is absent
BYTE for writing a byte, the java Byte class is mapped into a BYTE
BYTES for writing a sequence of bytes. It maps the java ByteBuffer class
INT for writing java integer
FLOAT for writing java float
LONG for writing java long
DOUBLE for writing java double
TEXT for writing java strings

BYTES is also used with everything else, for example, in Camel a file is sent around as an InputStream, int this case is written in a sequence file or a map file as a sequence of bytes.

Splitting Strategy
In the current version of Hadoop (0.20.1) opening a file in append mode is disabled since it’s not enough reliable. So, for the moment, it’s only possible to create new files. The Camel HDFS endpoint tries to solve this problem in this way:
• If the split strategy option has been defined, the actual file name will be <file name>0 initially
• Every time a splitting condition is met a new file is created with name <original file name>N where N is 1, 2, 3, etc.
The splitStrategy option is defined as a string with the following syntax:
splitStrategy=<ST>:<value>,<ST>:<value>,*

where <ST> can be:
BYTES a new file is created, and the old is closed when the number of written bytes is more than <value>
MESSAGES a new file is created, and the old is closed when the number of written messages is more than <value>
IDLE a new file is created, and the old is closed when no writing happened in the last <value> milliseconds

for example:
hdfs://localhost/tmp/simple-file?splitStrategy=IDLE:1000,BYTES:5
it means: a new file is created either when it has been idle for more than 1 second or if more than 5 bytes have been written