Merge branch 'devel'

etc/conf/jetty/webdefault.xml-auth

@@ -428,6 +428,34 @@
  <role-name>publisher</role-name>
  </auth-constraint>
  </security-constraint>
+
+ <security-constraint>
+ <web-resource-collection>
+ <web-resource-name>ESGF Solr</web-resource-name>
+ <url-pattern>/aggregations/admin/*</url-pattern>
+ </web-resource-collection>
+ <auth-constraint>
+ <role-name>admin</role-name>
+ </auth-constraint>
+ </security-constraint>
+ <security-constraint>
+ <web-resource-collection>
+ <web-resource-name>ESGF Solr</web-resource-name>
+ <url-pattern>/aggregations/update/*</url-pattern>
+ </web-resource-collection>
+ <auth-constraint>
+ <role-name>publisher</role-name>
+ </auth-constraint>
+ </security-constraint>
+ <security-constraint>
+ <web-resource-collection>
+ <web-resource-name>ESGF Solr</web-resource-name>
+ <url-pattern>/aggregations/delete/*</url-pattern>
+ </web-resource-collection>
+ <auth-constraint>
+ <role-name>publisher</role-name>
+ </auth-constraint>
+ </security-constraint>
 
  <login-config>
  <auth-method>DIGEST</auth-method>

etc/conf/wget-template

@@ -226,9 +226,9 @@ get_certificates() {
  [[ -z $force && "$(find $ESG_CERT_DIR -type d -mtime -1 2>/dev/null)" ]] && return 0
  echo -n "Retrieving Federation Certificates..." >&2
 
- if ! wget https://raw.github.com/ESGF/esgf-dist/master/installer/certs/esg_trusted_certificates.tar -O - -q | tar x -C $ESG_HOME; then
+ if ! wget --no-check-certificate https://raw.githubusercontent.com/ESGF/esgf-dist/master/installer/certs/esg_trusted_certificates.tar -O - -q | tar x -C $ESG_HOME; then
  #certificates tarred into esg_trusted_certificates. (if it breaks, let the user know why
- wget https://raw.github.com/ESGF/esgf-dist/master/installer/certs/esg_trusted_certificates.tar
+ wget --no-check-certificate https://raw.githubusercontent.com/ESGF/esgf-dist/master/installer/certs/esg_trusted_certificates.tar
  echo "Could't update certs!" >&2
  return 1
  else
@@ -247,9 +247,9 @@ get_truststore() {
  [[ -z $force && "$(find $TRUSTSTORE -type f -mtime -1 2>/dev/null)" ]] && return 0
  echo -n "Retrieving Federation Truststore..." >&2
 
- if ! wget https://raw.github.com/ESGF/esgf-dist/master/installer/certs/esg-truststore.ts -O $TRUSTSTORE -q ; then
+ if ! wget --no-check-certificate https://raw.githubusercontent.com/ESGF/esgf-dist/master/installer/certs/esg-truststore.ts -O $TRUSTSTORE -q ; then
  #certificates tarred into esg_trusted_certificates. (if it breaks, let the user know why
- wget https://raw.github.com/ESGF/esgf-dist/master/installer/certs/esg-truststore.ts -O $TRUSTSTORE
+ wget --no-check-certificate https://raw.githubusercontent.com/ESGF/esgf-dist/master/installer/certs/esg-truststore.ts -O $TRUSTSTORE
  echo "Could't update truststore!" >&2
  return 1
  else
@@ -271,7 +271,7 @@ get_credentials() {
  if [[ -z "$(find $MYPROXY_GETCERT -type f -mtime -1 2>/dev/null)" ]]; then
  echo -n "(Downloading $MYPROXY_GETCERT... "
  mkdir -p $(dirname $MYPROXY_GETCERT)
- if wget -q https://raw.github.com/ESGF/esgf-dist/master/installer/certs/getcert.jar -O $MYPROXY_GETCERT;then
+ if wget -q --no-check-certificate https://raw.githubusercontent.com/ESGF/esgf-dist/master/installer/certs/getcert.jar -O $MYPROXY_GETCERT;then
  echo 'done)'
  touch $MYPROXY_GETCERT
  else
@@ -376,7 +376,7 @@ find_credentials() {
  #some wget version complain if there's no file present
  [[ -f $COOKIE_JAR ]] || touch $COOKIE_JAR
 
- PKI_WGET_OPTS="$PKI_WGET_OPTS --certificate=$ESG_CERT --private-key=$ESG_KEY --save-cookies=$COOKIE_JAR --load-cookies=$COOKIE_JAR"
+ PKI_WGET_OPTS="$PKI_WGET_OPTS --certificate=$ESG_CERT --private-key=$ESG_KEY --save-cookies=$COOKIE_JAR --load-cookies=$COOKIE_JAR --ca-certificate=$ESG_CERT"
 
 }
 

src/java/main/esg/search/publish/plugins/FileDownloadHdfMetadataEnhancer.java

@@ -20,7 +20,7 @@
 import esg.search.query.api.QueryParameters;
 
 /**
- * HDF parser that download the file to a local cache 
+ * HDF parser that downloads the file to a local cache 
  * before parsing its content for metadata.
  * 
  * @author cinquini

src/java/main/esg/search/publish/plugins/HdfMetadataEnhancer.java

@@ -164,6 +164,9 @@ private void process(NetcdfFile ncfile, Record record) {
  }
  }
 
+ // file data format
+ record.setField(SolrXmlPars.FIELD_DATA_FORMAT, "HDF");
+
  }
 
  private Double[] parseCoordinate(NetcdfFile ncfile, String varName) {

src/java/main/esg/search/publish/security/AuthorizerAdapter.java

@@ -33,7 +33,7 @@ public class AuthorizerAdapter {
  private final Log LOG = LogFactory.getLog(this.getClass());
 
  // Note: comment out @Autowired to disable security
- //@Autowired // FIXME
+ @Autowired
  public void setAuthorizer(Authorizer authorizer) {
  this.authorizer = authorizer;
  }

src/java/main/esg/search/publish/thredds/ThreddsPars.java

@@ -111,7 +111,7 @@ public class ThreddsPars {
  </xsd:union>
  </xsd:simpleType>
  */
- public final static String SERVICE_TYPE_CATALOG = "Catalog";
+ public final static String SERVICE_TYPE_CATALOG = "THREDDS";
  public final static String SERVICE_TYPE_HTTP = "HTTPServer";
  public final static String SERVICE_TYPE_OPENDAP = "OpenDAP";
  public final static String SERVICE_TYPE_LAS = "LAS";

src/java/main/esg/search/publish/thredds/ThreddsParserStrategyTopLevelDatasetImpl.java

@@ -382,6 +382,7 @@ private Record newRecord(final InvDataset dataset, final boolean latest, final S
  // <dataset name="...." ID="..." restrictAccess="...">
  String id = dataset.getID().replaceAll("/", "."); // replace '/' in identifiers
  if (id.startsWith(".")) id = id.substring(1); // do not let id start with '.'
+ if (id.endsWith(".")) id = id.substring(0,id.length()-1); // do not let id end with '.'
  //String id = dataset.getID();
 
  // ...or assign random UUID if dataset id was not found

src/java/main/esg/search/query/impl/solr/SolrXmlPars.java

@@ -112,7 +112,7 @@ public class SolrXmlPars {
  final public static String FIELD_HEIGHT_UNITS = "height_units";
  final public static String FIELD_GEO = "geo";
 
- final public static String FIELD_DATA_FORMAT = "data_format";
+ final public static String FIELD_DATA_FORMAT = "format";
  final public static String FIELD_FILE_NAME = "file_name";
 
  /**

src/java/main/esg/search/utils/HttpClient.java

@@ -22,6 +22,7 @@
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.io.OutputStreamWriter;
+import java.net.HttpURLConnection;
 import java.net.URL;
 import java.net.URLConnection;
 
@@ -42,15 +43,25 @@ public class HttpClient {
  * @throws IOException
  */
  public String doGet(final URL url) throws IOException {
-
- // prepare HTTP request
- final URLConnection connection = url.openConnection(); 
- if (connectionTimeout!=0) connection.setConnectTimeout(connectionTimeout);
- if (readTimeout!=0) connection.setReadTimeout(readTimeout);
- connection.setUseCaches(false);
-
- // execute HTTP request
- return getResponse(connection);
+
+ HttpURLConnection connection = null;
+ try {
+
+ // prepare HTTP request
+ connection = (HttpURLConnection)url.openConnection();
+ if (connectionTimeout!=0) connection.setConnectTimeout(connectionTimeout);
+ if (readTimeout!=0) connection.setReadTimeout(readTimeout);
+ connection.setUseCaches(false);
+
+ // execute HTTP request
+ String response = getResponse(connection);
+ return response;
+
+ } finally {
+ if (connection != null) {
+ connection.disconnect();
+ }
+ }
 
  }
 
@@ -64,28 +75,40 @@ public String doGet(final URL url) throws IOException {
  */
  public String doPost(final URL url, final String data, boolean xml) throws IOException {
 
- // prepare HTTP request
- final URLConnection connection = url.openConnection();
- connection.setUseCaches(false);
- connection.setDoOutput(true); // POST method
- if (connectionTimeout!=0) connection.setConnectTimeout(connectionTimeout);
- if (readTimeout!=0) connection.setReadTimeout(readTimeout);
- if (xml) connection.setRequestProperty("Content-Type", "text/xml");
- connection.setRequestProperty("Charset", "utf-8");
-
- // preemptive authentication
- //final String userpassword = "<username>" + ":" + "<password>"; 
- //final byte[] authEncBytes = Base64.encodeBase64(userpassword.getBytes());
- //final String authStringEnc = new String(authEncBytes);
- //connection.setRequestProperty("Authorization", "Basic "+ authStringEnc ); 
-
- final OutputStreamWriter wr = new OutputStreamWriter(connection.getOutputStream());
- wr.write(data);
- wr.flush();
- wr.close();
+ HttpURLConnection connection = null;
+ OutputStreamWriter wr = null;
+
+ try {
+
+ // prepare HTTP request
+ connection = (HttpURLConnection)url.openConnection();
+ connection.setUseCaches(false);
+ connection.setDoOutput(true); // POST method
+ if (connectionTimeout!=0) connection.setConnectTimeout(connectionTimeout);
+ if (readTimeout!=0) connection.setReadTimeout(readTimeout);
+ if (xml) connection.setRequestProperty("Content-Type", "text/xml");
+ connection.setRequestProperty("Charset", "utf-8");
+
+ // send HTTP request
+ wr = new OutputStreamWriter(connection.getOutputStream());
+ wr.write(data);
+ wr.flush();
+
+ // receive HTTP response
+ String response = getResponse(connection);
+ return response;
 
- // execute HTTP request
- return getResponse(connection);
+ } finally { 
+ if (wr != null) {
+ try {
+ wr.close();
+ } catch (IOException e) {
+ }
+ }
+ if (connection != null) {
+ connection.disconnect();
+ }
+ }
 
  }
 
@@ -97,14 +120,28 @@ public String doPost(final URL url, final String data, boolean xml) throws IOExc
  */
  private String getResponse(final URLConnection connection) throws IOException {
 
- final BufferedReader rd = new BufferedReader(new InputStreamReader(connection.getInputStream()));
- final StringBuilder sb = new StringBuilder();
- String line = null;
- while ((line = rd.readLine()) != null) {
- sb.append(line + "\n");
+ BufferedReader rd = null;
+
+ try {
+
+ rd = new BufferedReader(new InputStreamReader(connection.getInputStream()));
+ final StringBuilder sb = new StringBuilder();
+ String line = null;
+ while ((line = rd.readLine()) != null) {
+ sb.append(line + "\n");
+ }
+ return sb.toString();
+
+ } finally {
+
+ if (rd != null) {
+ try {
+ rd.close();
+ } catch(Exception e) {
+ }
+ }
+
  }
- rd.close();
- return sb.toString();
 
  }