Skip to content

Commit

Permalink
Merge pull request #4 from RENCI-NRIG/outofmemoryfix
Browse files Browse the repository at this point in the history
Outofmemoryfix
  • Loading branch information
kthare10 authored Jul 24, 2019
2 parents fb8353b + 6f1ca32 commit 356177a
Show file tree
Hide file tree
Showing 15 changed files with 419 additions and 45 deletions.
5 changes: 5 additions & 0 deletions docker/clean.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
#!/bin/sh
docker stop $(docker ps -aq)
docker rm $(docker ps -aq)
docker rmi -f $(docker images -q)
docker network rm $(docker network ls --format "{{.ID}}")
20 changes: 20 additions & 0 deletions docker/config/log4j.properties
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
# default log level is fatal
# uncomment to enabled logging to stdout
#log4j.rootCategory = ERROR, stdout, file
#log4j.rootCategory = FATAL, file
log4j.rootLogger = WARN, stdout, file

# file logging
log4j.appender.file = org.apache.log4j.RollingFileAppender
log4j.appender.file.File = logs/imageService.log
log4j.appender.file.layout = org.apache.log4j.PatternLayout
log4j.appender.file.layout.ConversionPattern = %d-[%t]-{%-5p}-%c-(%F:%L)-%m%n
log4j.appender.file.MaxFileSize = 10MB
log4j.appender.file.MaxBackupIndex = 10
# stdout logging
log4j.appender.stdout = org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout = org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern = %d-[%t]-{%-5p}-%c-(%F:%L)-%m%n


log4j.category.org.renci.exogeni.image.service=DEBUG
98 changes: 86 additions & 12 deletions python/image_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,25 +37,45 @@ def __init__(self, log=None):
self._log = logging.getLogger('')

@classmethod
def _headers(self):
def get_image_size(self, host, project, user, password, imageId):
headers = {
'Accept': 'application/octet-stream',
'Accept': 'text/plain',
'Content-Type': "application/json",
}
return headers
params = {
'project':project,
'userName':user,
'password':password,
'imageId':imageId
}
response = requests.get((host + '/exoimageservice/1.0.0/imageSize'), headers=headers, params=params, stream=True, verify=False)
print ("Received Response Status: " + str(response.status_code))
return response

@classmethod
def get_image(self, host, project, user, password, imageId):
def get_image(self, host, project, user, password, imageId, contentRange=None):
headers = {
'Accept': 'application/octet-stream',
'Content-Type': "application/json",
}
if contentRange is not None:
headers = {
'Accept': 'application/octet-stream',
'Content-Type': "application/json",
'Range':contentRange
}

params = {
'project':project,
'userName':user,
'password':password,
'imageId':imageId
}
response = requests.get((host + '/exoimageservice/1.0.0/image'), headers=self._headers(), params=params, stream=True, verify=False)
response = requests.get((host + '/exoimageservice/1.0.0/image'), headers=headers, params=params, stream=True, verify=False)
print ("Received Response Status: " + str(response.status_code))
return response


def main():
parser = argparse.ArgumentParser(description='Python client to download images.\n')

Expand Down Expand Up @@ -108,19 +128,73 @@ def main():
help='file',
required=True
)
parser.add_argument(
'-s',
'--split',
dest='split',
type = bool,
help='download the file by parts',
required=False,
default=True
)
parser.add_argument(
'-r',
'--range',
dest='range',
type = int,
help='download the file by parts in increments of range',
required=False,
default=49999999
)

args = parser.parse_args()


im=ImageInterface()
r =im.get_image(args.exoimageHost, args.project, args.user, args.password, args.imageId)
print ("Headers: %s" % r.headers)
if r.status_code == 200:
with open(args.file, 'wb') as f:
r.raw.decode_content = True
shutil.copyfileobj(r.raw, f)

if args.split == True:
r =im.get_image_size(args.exoimageHost, args.project, args.user, args.password, args.imageId)
if r.status_code != 200 :
print ("Failed to determine image size")
sys.exit(1)
size=int(r.content)
print ("Image size: " + str(size))
start=0
end = args.range
length=0
while length < size :
contentRange='bytes=' + str(start) + "-" + str(end)
r =im.get_image(args.exoimageHost, args.project, args.user, args.password, args.imageId, contentRange)
print ("Downloaded: %s" % r.headers.get("Content-Range") + "; Image size: " + str(size) + " Content-Md5= %s" % r.headers.get("Content-Md5"))
#print ("Headers: %s" % r.headers)
if r.status_code == 200 or r.status_code == 206:
length = length + int(r.headers['Content-Length'])
with open(args.file, 'ab') as f:
r.raw.decode_content = True
shutil.copyfileobj(r.raw, f)
else :
print ("Failed to download image")
sys.exit(1)
start = end
end = start + args.range
start = start + 1
if r.status_code == 200:
break
else :
r =im.get_image(args.exoimageHost, args.project, args.user, args.password, args.imageId)
if r.status_code != 200 :
print ("Failed to determine image size")
sys.exit(1)
print ("Headers: %s" % r.headers)
if r.status_code == 200:
with open(args.file, 'wb') as f:
r.raw.decode_content = True
shutil.copyfileobj(r.raw, f)
else :
print ("Failed to download image")
sys.exit(1)
print ("file download complete")


sys.exit(0)

if __name__ == '__main__':
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
package org.renci.exogeni.image.service.api;

@javax.annotation.Generated(value = "io.swagger.codegen.v3.generators.java.SpringCodegen", date = "2019-05-29T10:20:00.231-04:00[America/New_York]")
@javax.annotation.Generated(value = "io.swagger.codegen.v3.generators.java.SpringCodegen", date = "2019-07-23T14:40:10.501-04:00[America/New_York]")

public class ApiException extends Exception{
private int code;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
import javax.servlet.*;
import javax.servlet.http.HttpServletResponse;

@javax.annotation.Generated(value = "io.swagger.codegen.v3.generators.java.SpringCodegen", date = "2019-05-29T10:20:00.231-04:00[America/New_York]")
@javax.annotation.Generated(value = "io.swagger.codegen.v3.generators.java.SpringCodegen", date = "2019-07-23T14:40:10.501-04:00[America/New_York]")

public class ApiOriginFilter implements javax.servlet.Filter {
@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

import javax.xml.bind.annotation.XmlTransient;

@javax.annotation.Generated(value = "io.swagger.codegen.v3.generators.java.SpringCodegen", date = "2019-05-29T10:20:00.231-04:00[America/New_York]")
@javax.annotation.Generated(value = "io.swagger.codegen.v3.generators.java.SpringCodegen", date = "2019-07-23T14:40:10.501-04:00[America/New_York]")

@javax.xml.bind.annotation.XmlRootElement
public class ApiResponseMessage {
Expand Down
21 changes: 11 additions & 10 deletions src/main/java/org/renci/exogeni/image/service/api/ImageApi.java
Original file line number Diff line number Diff line change
Expand Up @@ -22,22 +22,23 @@
import javax.validation.constraints.*;
import java.util.List;
import java.util.Map;
@javax.annotation.Generated(value = "io.swagger.codegen.v3.generators.java.SpringCodegen", date = "2019-06-07T10:26:14.705-04:00[America/New_York]")
@javax.annotation.Generated(value = "io.swagger.codegen.v3.generators.java.SpringCodegen", date = "2019-07-23T14:40:10.501-04:00[America/New_York]")

@Api(value = "image", description = "the image API")
public interface ImageApi {

@ApiOperation(value = "downloads image given image id", nickname = "imageGet", notes = "downloads image given image id ", response = String.class, tags={ })
@ApiResponses(value = {
@ApiResponse(code = 200, message = "OK", response = String.class),
@ApiResponse(code = 400, message = "Bad Request"),
@ApiResponse(code = 403, message = "Forbidden"),
@ApiResponse(code = 404, message = "Not Found"),
@ApiResponse(code = 500, message = "Internal Server Error"),
@ApiResponse(code = 200, message = "Unexpected Error") })
@ApiResponses(value = {
@ApiResponse(code = 200, message = "OK", response = String.class),
@ApiResponse(code = 206, message = "Partial Content", response = String.class),
@ApiResponse(code = 400, message = "Bad Request"),
@ApiResponse(code = 403, message = "Forbidden"),
@ApiResponse(code = 404, message = "Not Found"),
@ApiResponse(code = 500, message = "Internal Server Error"),
@ApiResponse(code = 200, message = "Unexpected Error") })
@RequestMapping(value = "/image",
produces = { "application/octet-stream" },
method = RequestMethod.GET)
produces = { "application/octet-stream" },
method = RequestMethod.GET)
ResponseEntity<InputStreamResource> imageGet(@NotNull @ApiParam(value = "", required = true) @Valid @RequestParam(value = "project", required = true) String project, @NotNull @ApiParam(value = "", required = true) @Valid @RequestParam(value = "userName", required = true) String userName, @NotNull @ApiParam(value = "", required = true) @Valid @RequestParam(value = "password", required = true) String password, @NotNull @ApiParam(value = "", required = true) @Valid @RequestParam(value = "imageId", required = true) String imageId);

}
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,9 @@
import javax.validation.Valid;
import javax.servlet.http.HttpServletRequest;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.List;
import java.util.Map;
@javax.annotation.Generated(value = "io.swagger.codegen.v3.generators.java.SpringCodegen", date = "2019-06-07T10:26:14.705-04:00[America/New_York]")
@javax.annotation.Generated(value = "io.swagger.codegen.v3.generators.java.SpringCodegen", date = "2019-07-23T14:40:10.501-04:00[America/New_York]")

@Controller
public class ImageApiController implements ImageApi {
Expand All @@ -45,19 +44,23 @@ public ImageApiController(ObjectMapper objectMapper, HttpServletRequest request)

public ResponseEntity<InputStreamResource> imageGet(@NotNull @ApiParam(value = "", required = true) @Valid @RequestParam(value = "project", required = true) String project, @NotNull @ApiParam(value = "", required = true) @Valid @RequestParam(value = "userName", required = true) String userName, @NotNull @ApiParam(value = "", required = true) @Valid @RequestParam(value = "password", required = true) String password, @NotNull @ApiParam(value = "", required = true) @Valid @RequestParam(value = "imageId", required = true) String imageId) {
String accept = request.getHeader("Accept");
String range = request.getHeader(HttpHeaders.RANGE);

OsImageController osImageController = null;
try {
osImageController = new OsImageController(project, userName, password);

OsImageController.ImageDetails imageDetails = osImageController.getImage(imageId);
OsImageController.ImageDetails imageDetails = osImageController.getImage(imageId, range);
ResponseEntity<String> responseEntity = new ResponseEntity<String>(HttpStatus.OK);


HttpHeaders headers = new HttpHeaders();
headers.add("Content-MD5", imageDetails.getMd5Checksum());
if(imageDetails.getRange() != null) {
headers.add("Content-Range", imageDetails.getRange());
}

return ResponseEntity.ok().headers(headers).contentLength(imageDetails.getLength())
return ResponseEntity.status(imageDetails.getStatus()).headers(headers).contentLength(imageDetails.getLength())
.contentType(MediaType.parseMediaType("application/octet-stream"))
.body(imageDetails.getInputStreamResource());
}
Expand All @@ -71,6 +74,7 @@ public ResponseEntity<InputStreamResource> imageGet(@NotNull @ApiParam(value = "
e.printStackTrace();
return new ResponseEntity<InputStreamResource>(HttpStatus.INTERNAL_SERVER_ERROR);
}

}

}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
/**
* NOTE: This class is auto generated by the swagger code generator program (3.0.3).
* https://github.com/swagger-api/swagger-codegen
* Do not edit the class manually.
*/
package org.renci.exogeni.image.service.api;

import io.swagger.annotations.*;
import org.springframework.http.ResponseEntity;
import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestHeader;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RequestPart;
import org.springframework.web.multipart.MultipartFile;

import javax.validation.Valid;
import javax.validation.constraints.*;
import java.util.List;
import java.util.Map;
@javax.annotation.Generated(value = "io.swagger.codegen.v3.generators.java.SpringCodegen", date = "2019-07-23T14:40:10.501-04:00[America/New_York]")

@Api(value = "imageSize", description = "the imageSize API")
public interface ImageSizeApi {

@ApiOperation(value = "returns image size given image id", nickname = "imageSizeGet", notes = "returns image size given image id ", response = String.class, tags={ })
@ApiResponses(value = {
@ApiResponse(code = 200, message = "OK", response = String.class),
@ApiResponse(code = 400, message = "Bad Request"),
@ApiResponse(code = 403, message = "Forbidden"),
@ApiResponse(code = 404, message = "Not Found"),
@ApiResponse(code = 500, message = "Internal Server Error"),
@ApiResponse(code = 200, message = "Unexpected Error") })
@RequestMapping(value = "/imageSize",
produces = { "text/plain" },
method = RequestMethod.GET)
ResponseEntity<String> imageSizeGet(@NotNull @ApiParam(value = "", required = true) @Valid @RequestParam(value = "project", required = true) String project,@NotNull @ApiParam(value = "", required = true) @Valid @RequestParam(value = "userName", required = true) String userName,@NotNull @ApiParam(value = "", required = true) @Valid @RequestParam(value = "password", required = true) String password,@NotNull @ApiParam(value = "", required = true) @Valid @RequestParam(value = "imageId", required = true) String imageId);

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
package org.renci.exogeni.image.service.api;

import com.fasterxml.jackson.databind.ObjectMapper;
import io.swagger.annotations.*;
import org.renci.exogeni.image.service.oscontroller.ImageServiceException;
import org.renci.exogeni.image.service.oscontroller.OsImageController;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestHeader;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RequestPart;
import org.springframework.web.multipart.MultipartFile;

import javax.validation.constraints.*;
import javax.validation.Valid;
import javax.servlet.http.HttpServletRequest;
import java.io.IOException;
import java.util.List;
import java.util.Map;
@javax.annotation.Generated(value = "io.swagger.codegen.v3.generators.java.SpringCodegen", date = "2019-07-23T14:40:10.501-04:00[America/New_York]")

@Controller
public class ImageSizeApiController implements ImageSizeApi {

private static final Logger log = LoggerFactory.getLogger(ImageSizeApiController.class);

private final ObjectMapper objectMapper;

private final HttpServletRequest request;

@org.springframework.beans.factory.annotation.Autowired
public ImageSizeApiController(ObjectMapper objectMapper, HttpServletRequest request) {
this.objectMapper = objectMapper;
this.request = request;
}

public ResponseEntity<String> imageSizeGet(@NotNull @ApiParam(value = "", required = true) @Valid @RequestParam(value = "project", required = true) String project,@NotNull @ApiParam(value = "", required = true) @Valid @RequestParam(value = "userName", required = true) String userName,@NotNull @ApiParam(value = "", required = true) @Valid @RequestParam(value = "password", required = true) String password,@NotNull @ApiParam(value = "", required = true) @Valid @RequestParam(value = "imageId", required = true) String imageId) {
String accept = request.getHeader("Accept");

OsImageController osImageController = null;
try {
osImageController = new OsImageController(project, userName, password);

Integer imageSize = osImageController.getImageDetails(imageId);
ResponseEntity<String> responseEntity = new ResponseEntity<String>(HttpStatus.OK);

return ResponseEntity.ok().contentType(MediaType.parseMediaType("text/plain")).body(imageSize.toString());
}
catch (ImageServiceException e) {
log.error("Exception occurred e=" + e);
e.printStackTrace();
return new ResponseEntity<String>(e.getStatus());
}
catch (Exception e) {
log.error("Exception occurred e=" + e);
e.printStackTrace();
return new ResponseEntity<String>(HttpStatus.INTERNAL_SERVER_ERROR);
}
}

}
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
package org.renci.exogeni.image.service.api;

@javax.annotation.Generated(value = "io.swagger.codegen.v3.generators.java.SpringCodegen", date = "2019-05-29T10:20:00.231-04:00[America/New_York]")
@javax.annotation.Generated(value = "io.swagger.codegen.v3.generators.java.SpringCodegen", date = "2019-07-23T14:40:10.501-04:00[America/New_York]")

public class NotFoundException extends ApiException {
private int code;
Expand Down
Loading

0 comments on commit 356177a

Please sign in to comment.