Update 2017-01-27: Since this commit there is a static method Repository::normalizeBranchName() that escapes a given string to form a valid ref name and returns it. The changes will be release with JGit v4.7.
For earlier versions of JGit, the name must be escaped manually. 
With Repository::isValidRefName() you can determine if the given string is a valid Git ref name. The documentation for git check-ref-format describes in detail the rules that a valid ref name must comply to.
I am using this method in a project that replaces all suspicious characters with an underscore (probably with some inaccuracies):
String DOT_LOCK = ".lock";
String REPLACEMENT = "_";
String escapeRefName( String refName ) {
  String result = refName;
  if( result.endsWith( DOT_LOCK ) ) {
    result = result.substring( 0, result.length() - DOT_LOCK.length() );
  }
  result = result.replace( " ", REPLACEMENT );
  result = result.replace( "\\", REPLACEMENT );
  result = result.replace( "/", REPLACEMENT );
  result = result.replace( "^", REPLACEMENT );
  result = result.replace( "@", REPLACEMENT );
  result = result.replace( "{", REPLACEMENT );
  result = result.replace( "}", REPLACEMENT );
  result = result.replace( "~", REPLACEMENT );
  result = result.replace( "*", REPLACEMENT );
  result = result.replace( "?", REPLACEMENT );
  result = result.replace( ":", REPLACEMENT );
  result = result.replace( "[", REPLACEMENT );
  result = result.replace( ".", REPLACEMENT );
  result = result.replace( "\u007F", REPLACEMENT );
  return result;
}