This Google App Engine (GAE) app receives export requests at `/cloud-datastore-export` and

sends an export request to the [Cloud Datastore Admin API](https://cloud.google.com/datastore/docs/reference/admin/rest/v1/projects/export).

This app requires the following to complete export operations:

1. A Google Cloud project with billing enabled.

1. A Cloud Storage bucket for your export files.

1. The App Engine default service account must have permission

to write to the Cloud Storage bucket and have the Cloud Datastore Import Export Admin IAM role.

For more information on completing these requirements, see the

[Cloud Datastore documentation](https://cloud.google.com/datastore/docs/schedule-export#before_you_begin).

Set the target project in gcloud:

gcloud config set project PROJECT_NAME

Deploy the GAE app:

mvn appengine:deploy

The app takes the following parameters:

* `output_url_prefix` (required)-specifies where to save your Cloud Datastore export. If the URL ends with a `/`, it's used as is. Otherwise, the app adds a timesamp to the url.

* `kind` (optional, multiple)-restricts export to only these kinds.

* `namespace_id` (optional, multiple)-restricts export to only these namespaces.

Modify and deploy the cronjob:

gcloud app deploy cron.yaml

You can test your cron job by running the job manually:

<a href="https://console.cloud.google.com/appengine/cronjobs">Open the Cron Jobs page</a>

cron:

- description: "Daily Cloud Datastore Export"

url: /cloud-datastore-export?output_url_prefix=gs://<EXPORT_BUCKET>&kind=<KIND_NAME>

schedule: every 24 hours

<?xml version="1.0" encoding="UTF-8"?>

<project>

<modelVersion>4.0.0</modelVersion>

<packaging>war</packaging>

<version>1.0-SNAPSHOT</version>

<groupId>com.example.datastore</groupId>

<artifactId>schedule-export</artifactId>

<parent>

<groupId>com.google.cloud.samples</groupId>

<artifactId>shared-configuration</artifactId>

<version>1.0.10</version>

</parent>

<properties>

<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>

<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>

<maven.compiler.source>1.8</maven.compiler.source>

<maven.compiler.target>1.8</maven.compiler.target>

<maven.compiler.showDeprecation>true</maven.compiler.showDeprecation>

<archiveClasses>true</archiveClasses>

</properties>

<prerequisites>

<maven>3.5</maven>

</prerequisites>

<dependencies>

<!-- Compile/runtime dependencies -->

<dependency>

<groupId>com.google.appengine</groupId>

<artifactId>appengine-api-1.0-sdk</artifactId>

<version>1.9.59</version>

</dependency>

<dependency>

<groupId>com.google.guava</groupId>

<artifactId>guava</artifactId>

<version>25.1-android</version>

</dependency>

<dependency>

<groupId>javax.servlet</groupId>

<artifactId>javax.servlet-api</artifactId>

<version>3.1.0</version>

<type>jar</type>

<scope>provided</scope>

</dependency>

<dependency>

<groupId>org.json</groupId>

<artifactId>json</artifactId>

<version>20180130</version>

</dependency>

<!-- Test Dependencies -->

<dependency>

<groupId>com.google.appengine</groupId>

<artifactId>appengine-testing</artifactId>

<version>1.9.59</version>

<scope>test</scope>

</dependency>

<dependency>

<groupId>com.google.truth</groupId>

<artifactId>truth</artifactId>

<version>0.33</version>

<scope>test</scope>

</dependency>

<dependency>

<groupId>junit</groupId>

<artifactId>junit</artifactId>

<version>4.12</version>

<scope>test</scope>

</dependency>

<dependency>

<groupId>org.mockito</groupId>

<artifactId>mockito-all</artifactId>

<version>2.0.2-beta</version>

<scope>test</scope>

</dependency>

</dependencies>

<build>

<!-- for hot reload of the web application-->

<outputDirectory>${project.build.directory}/${project.build.finalName}/WEB-INF/classes</outputDirectory>

<plugins>

<plugin>

<groupId>com.google.cloud.tools</groupId>

<artifactId>appengine-maven-plugin</artifactId>

<version>1.3.2</version>

<configuration>

</configuration>

</plugin>

</plugins>

</build>

</project>

package com.google.example.datastore;

import com.google.appengine.api.appidentity.AppIdentityService;

import com.google.appengine.api.appidentity.AppIdentityServiceFactory;

import com.google.apphosting.api.ApiProxy;

import com.google.common.io.CharStreams;

import java.io.IOException;

import java.io.InputStream;

import java.io.InputStreamReader;

import java.io.OutputStreamWriter;

import java.net.HttpURLConnection;

import java.net.URL;

import java.nio.charset.StandardCharsets;

import java.text.SimpleDateFormat;

import java.util.ArrayList;

import java.util.Date;

import java.util.logging.Logger;

import javax.servlet.annotation.WebServlet;

import javax.servlet.http.HttpServlet;

import javax.servlet.http.HttpServletRequest;

import javax.servlet.http.HttpServletResponse;

import org.json.JSONArray;

import org.json.JSONObject;

import org.json.JSONTokener;

@WebServlet(name = "DatastoreExportServlet", value = "/cloud-datastore-export")

public class DatastoreExportServlet extends HttpServlet {

private static final String PROJECT_ID = System.getenv("GOOGLE_CLOUD_PROJECT");

private static final Logger log = Logger.getLogger(DatastoreExportServlet.class.getName());

@Override

public void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException {

// Validate outputURL parameter

String outputUrlPrefix = request.getParameter("output_url_prefix");

if (outputUrlPrefix == null || !outputUrlPrefix.matches("^gs://.*")) {

// Send error response if outputURL not set or not a Cloud Storage bucket

response.setStatus(HttpServletResponse.SC_CONFLICT);

response.setContentType("text/plain");

response.getWriter().println("Error: Must provide a valid output_url_prefix.");

} else {

// Put together export request headers

URL url = new URL("https://datastore.googleapis.com/v1/projects/" + PROJECT_ID + ":export");

HttpURLConnection connection = (HttpURLConnection) url.openConnection();

connection.setDoOutput(true);

connection.setRequestMethod("POST");

connection.addRequestProperty("Content-Type", "application/json");

// Get an access token to authorize export request

ArrayList<String> scopes = new ArrayList<String>();

scopes.add("https://www.googleapis.com/auth/datastore");

final AppIdentityService appIdentity = AppIdentityServiceFactory.getAppIdentityService();

final AppIdentityService.GetAccessTokenResult accessToken =

AppIdentityServiceFactory.getAppIdentityService().getAccessToken(scopes);

connection.addRequestProperty("Authorization", "Bearer " + accessToken.getAccessToken());

// Build export request payload based on URL parameters

// Required: output_url_prefix

// Optional: entity filter

JSONObject exportRequest = new JSONObject();

// If output prefix ends with a slash, use as-is

// Otherwise, add a timestamp to form unique output url

if (!outputUrlPrefix.endsWith("/")) {

String timeStamp = new SimpleDateFormat("yyyyMMddHHmmss").format(new Date());

outputUrlPrefix = outputUrlPrefix + "/" + timeStamp + "/";

}

// Add outputUrl to payload

exportRequest.put("output_url_prefix", outputUrlPrefix);

// Build optional entity filter to export subset of

// kinds or namespaces

JSONObject entityFilter = new JSONObject();

// Read kind parameters and add to export request if not null

String[] kinds = request.getParameterValues("kind");

if (kinds != null) {

JSONArray kindsJson = new JSONArray(kinds);

entityFilter.put("kinds", kinds);

}

// Read namespace parameters and add to export request if not null

String[] namespaces = request.getParameterValues("namespace_id");

if (namespaces != null) {

JSONArray namespacesJson = new JSONArray(namespaces);

entityFilter.put("namespaceIds", namespacesJson);

}

// Add entity filter to payload

// Finish export request payload

exportRequest.put("entityFilter", entityFilter);

// Send export request

OutputStreamWriter writer = new OutputStreamWriter(connection.getOutputStream());

exportRequest.write(writer);

writer.close();

// Examine server's response

if (connection.getResponseCode() != HttpURLConnection.HTTP_OK) {

// Request failed, log errors and return

InputStream s = connection.getErrorStream();

InputStreamReader r = new InputStreamReader(s, StandardCharsets.UTF_8);

String errorMessage =

String.format(

"got error (%d) response %s from %s",

connection.getResponseCode(), CharStreams.toString(r), connection.toString());

log.warning(errorMessage);

response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);

response.setContentType("text/plain");

response.getWriter().println(

"Failed to initiate export.");

return;

}

// Success, print export operation information

JSONObject exportResponse = new JSONObject(new JSONTokener(connection.getInputStream()));

response.setContentType("text/plain");

response.getWriter().println(

"Export started:\n" + exportResponse.toString(4));

}

}

}

<?xml version="1.0" encoding="utf-8"?>

<appengine-web-app xmlns="http://appengine.google.com/ns/1.0">

<runtime>java8</runtime>

<threadsafe>true</threadsafe>

<system-properties>

<property name="java.util.logging.config.file" value="WEB-INF/logging.properties"/>

</system-properties>

</appengine-web-app>

.level = WARNING

<?xml version="1.0" encoding="utf-8"?>

<web-app xmlns="http://xmlns.jcp.org/xml/ns/javaee"

xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"

xsi:schemaLocation="http://xmlns.jcp.org/xml/ns/javaee

http://xmlns.jcp.org/xml/ns/javaee/web-app_3_1.xsd" version="3.1">

<welcome-file-list>

<welcome-file>index.jsp</welcome-file>

</welcome-file-list>

</web-app>

<!DOCTYPE html>

<%@ page contentType="text/html;charset=UTF-8" language="java" %>

<html>

<head>

<link href='//fonts.googleapis.com/css?family=Marmelad' rel='stylesheet' type='text/css'>

<title>Hello App Engine Standard Java 8</title>

</head>

<body>

<h1>Hello App Engine!</h1>

</body>

</html>